From ff6bb585d806261886666786e837cd21a0e289d0 Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.8 (1M context)" Date: Sat, 18 Jul 2026 14:06:36 -0700 Subject: [PATCH 1/7] fix(security): canonical secret detector catches distinctive-prefix credentials glued after a word char (leading \b evasion) across audit/realtime/memory egress sinks; bounded to high-entropy whole-match prefixes with a benign-identifier no-over-redaction corpus Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01JhCDTLngxAf6MDh4Bfcd48 --- src/security/friday-secret-shape-redactor.ts | 76 +++++++++-- .../friday-event-payload-redactor.test.ts | 20 +++ ...friday-memory-output-filter-secret.test.ts | 44 ++++++ .../friday-secret-shape-redactor.test.ts | 129 +++++++++++++++++- ...nt-redaction-001-cross-sink-parity.test.ts | 38 ++++++ 5 files changed, 291 insertions(+), 16 deletions(-) diff --git a/src/security/friday-secret-shape-redactor.ts b/src/security/friday-secret-shape-redactor.ts index 95310b687..21417f738 100644 --- a/src/security/friday-secret-shape-redactor.ts +++ b/src/security/friday-secret-shape-redactor.ts @@ -182,19 +182,36 @@ const SECRET_CONTENT_PATTERNS: readonly SecretContentPattern[] = [ sensitiveSpan: (m) => credentialAfterPrefix(m, m[1]), }, // JWT — three base64url segments, the first beginning `eyJ` (`{"…`). + // GLUED-PREFIX: leading `\b` KEPT deliberately. `eyJ` is not a delimiter-prefix — it is the base64url + // of `{"` and thus part of the token BODY (base64url includes `_`/`-`), so dropping the boundary would + // let `eyJ` match mid-base64-blob and newly over-redact benign base64 (a case the no-degrade corpus + // guards). The boundary stays; a JWT glued after a word char is out of this fix's enumerated scope. { pattern: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{2,}\.[A-Za-z0-9_-]{10,}\b/gu, sensitiveSpan: wholeMatchSpan, }, // GitHub tokens: classic `gh?_` prefixes AND fine-grained `github_pat_`. + // + // GLUED-PREFIX (SEC-SECRET-GLUED-PREFIX-001): the classic `gh[opsru]_` branch drops its leading `\b` + // so a token glued to a preceding word char (`keyghp_<16+>`) is still caught — no benign identifier + // segment ends in `ghp`/`gho`/`ghs`/`ghr`/`ghu` immediately before `_` (there is no such English / + // snake_case fragment), so allowing a glued prefix cannot over-match. The `github_pat_` branch KEEPS + // its leading `\b` (moved onto the branch): `github_pat` is a NATURAL English phrase ("github personal + // access token"), so a benign flat identifier `stored_github_pat_reference_token` + a `[A-Za-z0-9_]` + // body (which INCLUDES `_`) could plausibly appear in config/source — the boundary keeps it protected. { - pattern: /\b(?:gh[opsru]_[A-Za-z0-9_]{16,}|github_pat_[A-Za-z0-9_]{16,})\b/gu, + pattern: /(?:gh[opsru]_[A-Za-z0-9_]{16,}|\bgithub_pat_[A-Za-z0-9_]{16,})\b/gu, sensitiveSpan: wholeMatchSpan, }, // Provider hyphen-prefixed API keys: OpenAI `sk-` / `sk-proj-`, `rk-`, `ak-`, xAI `xai-`. The // client-safe PUBLISHABLE `pk-` prefix is DELIBERATELY ABSENT from the alternation (round-16): a // Stripe/Google publishable key is not a secret, and Friday's own domain uses `pk-` for a satellite // PUBLIC key (`publicKey: "pk-…"`), so redacting a `pk-` value is data-loss, not protection. + // GLUED-PREFIX: leading `\b` KEPT deliberately. `sk`/`rk`/`ak` are SHORT (2-char) prefixes that glue + // into extremely common words ending in them — `desk-`, `risk-`, `task-`, `disk-`, `mask-`, `work-`, + // `mark-`, `fork-`, `network-`, `break-`, `speak-`, `leak-` — and the `-`-inclusive body would then + // over-redact a benign kebab identifier (`desk-management-framework-v2` → `sk-management-framework-v2`). + // The boundary is REQUIRED here; this is the canonical "short/low-signal prefix" exclusion. { pattern: /\b(?:sk|rk|ak|xai)-(?:proj-)?[A-Za-z0-9_-]{16,}\b/gu, sensitiveSpan: wholeMatchSpan, @@ -206,6 +223,10 @@ const SECRET_CONTENT_PATTERNS: readonly SecretContentPattern[] = [ // key never matches. (Stripe key bodies are base62 with no internal `_`, so the value run is // `[A-Za-z0-9]{16,}`; the leading `(?:sk|rk)_(?:live|test)` / `whsec` cannot false-fire on benign // snake_case identifiers.) + // GLUED-PREFIX: leading `\b` KEPT deliberately. The `sk_`/`rk_` prefixes glue into the SAME common + // words as the hyphen form (`desk_live_…`, `risk_test_…`, `task_live_…`), so — like `sk-`/`rk-` — the + // boundary is retained; catching a glued Stripe underscore-key is not worth the benign-`sk_`/`rk_` + // over-redaction risk. (The distinctive `whsec_` shares this pattern and inherits the kept boundary.) { pattern: /\b(?:(?:sk|rk)_(?:live|test)|whsec)_[A-Za-z0-9]{16,}\b/gu, sensitiveSpan: wholeMatchSpan, @@ -213,11 +234,18 @@ const SECRET_CONTENT_PATTERNS: readonly SecretContentPattern[] = [ // AWS access-key id (`AKIA…` long-term + STS/temporary variants). NB: the AWS SECRET access key is // a shapeless 40-char base64 with no prefix — it is caught by its KEY NAME (`secretAccessKey`, // in the sensitive-field-name set), NOT a shape, so it is not (and must not be) a content pattern. + // GLUED-PREFIX: leading `\b` dropped (see hf_ note). The `A(?:KIA|SIA|…)` prefix + 16 CONTIGUOUS + // `[0-9A-Z]` (uppercase/digit only — a `_`, `-`, or lowercase char breaks the body) is the gate: a + // benign snake_case constant like `REGION_ASIA_PACIFIC_1` cannot span it (the `_` after `ASIA` ends + // the body at 0 chars), so allowing a glued prefix (`xAKIA<16>`) does not over-match plausible ids. { - pattern: /\bA(?:KIA|SIA|GPA|IDA|ROA|IPA|NPA|NVA)[0-9A-Z]{16}\b/gu, + pattern: /A(?:KIA|SIA|GPA|IDA|ROA|IPA|NPA|NVA)[0-9A-Z]{16}\b/gu, sensitiveSpan: wholeMatchSpan, }, // Google API key — `AIza` + 35 chars of `[0-9A-Za-z_-]` (documented 39-char format). + // GLUED-PREFIX: leading `\b` KEPT deliberately. `AIza` begins with `AI`, a ubiquitous fragment in an + // AI-domain codebase (`openAI…`, `vertexAI…`), so a glued form (`openAIza<35>`) is a plausible-enough + // benign collision that the NO-DEGRADE hard bar favors the boundary over catching a (rare) glued key. { pattern: /\bAIza[0-9A-Za-z_-]{35}\b/gu, sensitiveSpan: wholeMatchSpan, @@ -227,42 +255,53 @@ const SECRET_CONTENT_PATTERNS: readonly SecretContentPattern[] = [ // distinctive). No trailing `\b` — the body legitimately ends in `.`/`-`/`_`; the greedy run grabs // the whole token and the marker replaces it. A benign near-miss `ya29_notatoken` (underscore, not // the required literal `.`) never matches. + // GLUED-PREFIX: leading `\b` KEPT deliberately. A username-style identifier ending in `ya29` + // (`maya29`, `sonya29`, `priya29`) followed by `.` would collide (`maya29.profile_image_url` + // → `ya29.profile_image_url`), so the boundary is retained over catching a (rare) glued access token. { pattern: /\bya29\.[A-Za-z0-9._-]{20,}/gu, sensitiveSpan: wholeMatchSpan, }, // Google OAuth CLIENT SECRET — `GOCSPX-` + a base64url body (distinctive prefix, low false positive). + // GLUED-PREFIX: leading `\b` dropped — `GOCSPX-` is an all-caps, non-word prefix (no benign glue). { - pattern: /\bGOCSPX-[A-Za-z0-9_-]{20,}\b/gu, + pattern: /GOCSPX-[A-Za-z0-9_-]{20,}\b/gu, sensitiveSpan: wholeMatchSpan, }, // GitLab personal access token — `glpat-` + 20+ base64url chars (distinctive prefix). A benign // `glpat_docs` (underscore, not the required hyphen) never matches. + // GLUED-PREFIX: leading `\b` dropped — no benign identifier segment ends in `glpat` before a `-`. { - pattern: /\bglpat-[A-Za-z0-9_-]{20,}\b/gu, + pattern: /glpat-[A-Za-z0-9_-]{20,}\b/gu, sensitiveSpan: wholeMatchSpan, }, // SendGrid API key — the exact `SG.<22>.<43>` structure (two dot-separated base64url runs of fixed // length). The structured length is what makes `SG.` specific enough to avoid a benign-text false fire. + // GLUED-PREFIX: leading `\b` dropped — the rigid `SG..` shape is what gates + // it, so a glued `imgSG.<22>.<43>` cannot arise from a plausible benign identifier. { - pattern: /\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b/gu, + pattern: /SG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b/gu, sensitiveSpan: wholeMatchSpan, }, // Square access token (`sq0atp-`) / OAuth client secret (`sq0csp-`) — distinctive `sq0` prefix. The // newer `EAAA…` form is DELIBERATELY excluded (4-char base64 prefix, unacceptable false-positive risk). + // GLUED-PREFIX: leading `\b` dropped — `sq0atp-`/`sq0csp-` are not benign identifier fragments. { - pattern: /\bsq0(?:atp|csp)-[A-Za-z0-9_-]{22,60}\b/gu, + pattern: /sq0(?:atp|csp)-[A-Za-z0-9_-]{22,60}\b/gu, sensitiveSpan: wholeMatchSpan, }, // DigitalOcean personal access token — `dop_v1_` + exactly 64 hex (distinctive prefix + fixed length). + // GLUED-PREFIX: leading `\b` dropped — `dop_v1_` + EXACTLY 64 lowercase-hex is astronomically specific. { - pattern: /\bdop_v1_[a-f0-9]{64}\b/gu, + pattern: /dop_v1_[a-f0-9]{64}\b/gu, sensitiveSpan: wholeMatchSpan, }, // Groq API key — `gsk_` + 40+ base62 chars (Friday's provider-catalog classifies `gsk_` as Groq with // HIGH confidence; the body class excludes `_`, so a benign `gsk_` snake_case identifier never matches). + // GLUED-PREFIX: leading `\b` dropped — see the hf_ note. Body `[A-Za-z0-9]{40,}` EXCLUDES `_`/`-`, + // so a snake_case run cannot span it and `gsk` is not a benign fragment; a glued `xgsk_<40>` is caught. { - pattern: /\bgsk_[A-Za-z0-9]{40,}\b/gu, + pattern: /gsk_[A-Za-z0-9]{40,}\b/gu, sensitiveSpan: wholeMatchSpan, }, // HuggingFace user access token — `hf_` + 34+ base62 chars (Friday's provider-catalog classifies `hf_` @@ -272,20 +311,35 @@ const SECRET_CONTENT_PATTERNS: readonly SecretContentPattern[] = [ // open-ended (NOT exact `{34}`) so a longer fine-grained token cannot slip the trailing `\b` and leak, // mirroring the `gsk_`/`glpat-` open-ended convention. The body class excludes `_`, so a benign short // `hf_docs` / `hf_` / `hf_config_value` snake_case identifier never matches (distinctive, low false-positive). + // + // GLUED-PREFIX (SEC-SECRET-GLUED-PREFIX-001): the leading `\b` is DROPPED so a credential glued to a + // preceding ASCII word char (`keyhf_<34>`) — which has NO word boundary before `hf` — is still caught. + // This is safe ONLY because the body is long + high-entropy: `[A-Za-z0-9]{34,}` EXCLUDES `_` and `-`, + // so it cannot span a snake_case / kebab identifier boundary, and no benign identifier is realistically + // going to contain the literal `hf_` followed by 34+ contiguous base62 chars. The trailing `\b` (and + // the whole-match span) are UNCHANGED, so the redacted span is exactly `hf_` and the benign + // leading char (`key`) survives byte-for-byte. Same reasoning applies to every de-`\b`'d shape below/above. { - pattern: /\bhf_[A-Za-z0-9]{34,}\b/gu, + pattern: /hf_[A-Za-z0-9]{34,}\b/gu, sensitiveSpan: wholeMatchSpan, }, // npm access token — `npm_` + 36 base62 chars. The body class excludes `_`, so a benign `npm_` // config identifier (`npm_config_cache`) never matches (it is short and contains `_`). + // GLUED-PREFIX: leading `\b` dropped — body `[A-Za-z0-9]{36}` excludes `_`, `npm` is not a benign + // fragment ending a word before `_`, so a glued `xnpm_<36>` is caught without over-matching. { - pattern: /\bnpm_[A-Za-z0-9]{36}\b/gu, + pattern: /npm_[A-Za-z0-9]{36}\b/gu, sensitiveSpan: wholeMatchSpan, }, // Slack tokens: bot/user `xox[abprs]-` AND app-level `xapp-` (Socket Mode). Friday's own Slack setup // recipe marks the `xapp-` app-level token `sensitive: true`; the `xapp-` prefix is distinctive. + // + // GLUED-PREFIX: the leading `\b` is DROPPED for the `xox[abprs]-` bot/user branch only (`xoxb`/`xoxp`/… + // are not benign identifier fragments, so a glued `zxoxb-<10+>` is caught). The `xapp-` branch KEEPS its + // leading `\b` (moved onto the branch): `app` is a ubiquitous fragment and `xapp` glues into plausible + // benign identifiers (`maxapp-…`, `linuxapp-…`), which a `-`-inclusive body would then over-redact. { - pattern: /\b(?:xox[abprs]|xapp)-[A-Za-z0-9-]{10,}\b/gu, + pattern: /(?:xox[abprs]|\bxapp)-[A-Za-z0-9-]{10,}\b/gu, sensitiveSpan: wholeMatchSpan, }, // Generic credential ASSIGNMENT embedded in free text: `=` / `: `. diff --git a/test/unit/api/realtime/friday-event-payload-redactor.test.ts b/test/unit/api/realtime/friday-event-payload-redactor.test.ts index 92a130b39..9a16fb428 100644 --- a/test/unit/api/realtime/friday-event-payload-redactor.test.ts +++ b/test/unit/api/realtime/friday-event-payload-redactor.test.ts @@ -157,6 +157,26 @@ describe("redactEventPayload — secret shapes (no regression)", () => { expect(s).toContain("[REDACTED]"); }); + // SEC-SECRET-GLUED-PREFIX-001: a distinctive-prefix credential GLUED directly to a preceding word + // char (`keyhf_<34>`) had no word boundary before the prefix, so the canonical detector's leading + // `\b` skipped it and it egressed on-wire to agents/channels. Built from parts (`seg`) so no literal + // token appears in SOURCE. RED on bf6968f9 (credential survives in the payload), GREEN after. + it("redacts a distinctive-prefix credential GLUED after a word char (glued-prefix evasion, on-wire)", () => { + const seg = (...p: string[]) => p.join(""); // pragma: allowlist secret + const HF = seg("hf_", "AbCdEfGhIjKlMnOpQrStUvWxYz01234567"); // pragma: allowlist secret — 34 base62 + const GSK = seg("gsk_", "abcdefghijklmnopqrstuvwxyz0123456789ABCDwx"); // pragma: allowlist secret — 42 base62 + const GLPAT = seg("glpat-", "ABCdef0123456789ghijkLMNop"); // pragma: allowlist secret + const out = redactEventPayload({ + log: seg("deploy key", HF, " and x", GSK, " and id", GLPAT, " done"), + }); + const s = serialize(out); + for (const cred of [HF, GSK, GLPAT]) expect(s, cred).not.toContain(cred); + expect(s).toContain("[REDACTED]"); + // Benign surrounding + glued leading chars survive (only the credential subspan is masked). + expect(s).toContain("deploy key"); + expect(s).toContain("done"); + }); + it("masks values under legacy sensitive keys", () => { const out = redactEventPayload({ password: "hunter2-a5", // pragma: allowlist secret diff --git a/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts b/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts index 898ed8674..7cf27bccb 100644 --- a/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts +++ b/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts @@ -271,3 +271,47 @@ describe("FridayMemoryOutputFilter — round-17 HuggingFace hf_ shape (content + expect(out.content).toBe("hf_docs reference"); }); }); + +// ─── SEC-SECRET-GLUED-PREFIX-001: a distinctive-prefix credential GLUED directly to a preceding ASCII +// word char (`keyhf_<34>`) had no word boundary before the prefix, so the canonical detector's +// leading `\b` skipped it and it survived memory read-back to agents through EVERY output-filter leg +// (content scanAndTransform / metadata redactDeep / tag drop). RED on bf6968f9 (glued token verbatim +// in content + metadata, kept as a tag); GREEN after the leading `\b` is dropped on the high-entropy +// distinctive-prefix patterns. Built from parts so no contiguous literal token appears in SOURCE. ─── +describe("FridayMemoryOutputFilter — SEC-SECRET-GLUED-PREFIX-001 glued distinctive-prefix credential", () => { + const filter = createFridayMemoryOutputFilter(); + const seg = (...p: string[]): string => p.join(""); // pragma: allowlist secret + const HF_BODY = "AbCdEfGhIjKlMnOpQrStUvWxYz01234567"; // pragma: allowlist secret — 34 base62 + const HF = seg("hf_", HF_BODY); // pragma: allowlist secret + const GLUED = seg("key", HF); // pragma: allowlist secret — hf_ glued after the word `key` + + it("redacts a GLUED hf_ token in item.content (scanAndTransform leg), leaving the benign leading `key`", () => { + const out = filter.filterItem(makeItem({ content: seg("auth used ", GLUED, " today") })); + expect(out.content).toContain(M); + expect(out.content).not.toContain(HF); + expect(out.content).not.toContain(HF_BODY); + // Only the credential subspan is masked — the glued `key` and surrounding text survive. + expect(out.content).toBe(`auth used key${M} today`); + }); + + it("redacts a GLUED hf_ token carried in item.metadata VALUES (redactDeep leg) under a non-sensitive key", () => { + const out = filter.filterItem(makeItem({ metadata: { tokenPreview: GLUED, note: "keep" } })); + const md = out.metadata as { tokenPreview: string; note: string }; + expect(md.tokenPreview).toBe(`key${M}`); + expect(md.note).toBe("keep"); + expect(JSON.stringify(md)).not.toContain(HF_BODY); + }); + + it("DROPS a GLUED hf_-shaped tag, preserving benign tags byte-identical", () => { + const out = filter.filterItem(makeItem({ tags: ["ok", GLUED, "fine"] })); + expect(JSON.stringify(out.tags)).not.toContain(HF_BODY); + expect(out.tags).toEqual(["ok", "fine"]); + }); + + it("NO-DEGRADE: a word-embedded near-miss with a SHORT body is not a token (survives)", () => { + // `keyhf_` + a short body has no 34-char high-entropy run, so it is NOT a credential — untouched. + const out = filter.filterItem(makeItem({ tags: [seg("key", "hf_docs")], content: seg("ref key", "hf_docs") })); + expect(out.tags).toEqual([seg("key", "hf_docs")]); + expect(out.content).toBe(seg("ref key", "hf_docs")); + }); +}); diff --git a/test/unit/security/friday-secret-shape-redactor.test.ts b/test/unit/security/friday-secret-shape-redactor.test.ts index 99087a0bd..362d16c37 100644 --- a/test/unit/security/friday-secret-shape-redactor.test.ts +++ b/test/unit/security/friday-secret-shape-redactor.test.ts @@ -385,16 +385,18 @@ describe("friday-secret-shape-redactor", () => { expect(spans[0]!.replacement).toBe(M); }); - // NO-DEGRADE: the body class excludes `_` and requires a leading word boundary + 34+ chars, so a - // benign `hf_`-prefixed snake_case identifier, a short form, a bare 34-char hash (no prefix), and a - // mid-word `…hf_…` all survive byte-identical with NO span (not an over-redactor). - it("does NOT redact benign hf_ near-misses (hf_docs / short / underscore body / bare hash / mid-word)", () => { + // NO-DEGRADE: the body class excludes `_` and requires 34+ contiguous base62 chars, so a benign + // `hf_`-prefixed snake_case identifier, a short form, and a bare 34-char hash (no prefix) all survive + // byte-identical with NO span (not an over-redactor). NOTE: a `hf_` GLUED to a preceding word char + // followed by a real 34+ high-entropy body (`shf_<34>`) is NO LONGER a benign near-miss — it is a + // leaked-credential evasion the glued-prefix fix (SEC-SECRET-GLUED-PREFIX-001) now catches; that case + // moved to the glued-prefix describe block below. + it("does NOT redact benign hf_ near-misses (hf_docs / short / underscore body / bare hash)", () => { for (const benign of [ "hf_docs", // short (4 body chars) "hf_", // prefix only "hf_config_value_thing", // underscore-separated identifier, no 34-char base62 run "AbCdEfGhIjKlMnOpQrStUvWxYz01234567", // pragma: allowlist secret — bare 34-char hash, NO hf_ prefix - seg("shf_", HF_BODY), // pragma: allowlist secret — no word boundary before `hf` (embedded in a longer token) ]) { expect(redactSecretShapesInString(benign), benign).toBe(benign); expect(findSecretShapeSpans(benign), benign).toEqual([]); @@ -406,4 +408,121 @@ describe("friday-secret-shape-redactor", () => { expect(redactSecretShapesInString(HF)).not.toContain(HF); }); }); + + // ─── SEC-SECRET-GLUED-PREFIX-001: the canonical detector missed a distinctive-prefix credential + // GLUED directly after an ASCII word char (`keyhf_<34>`): the leading `\b` on the whole-match + // prefix patterns requires a NON-word boundary before the prefix, so `hf_` has no + // boundary and survived — reaching the audit-read, realtime on-wire, and memory read-back sinks. + // The fix DROPS the leading `\b` on the HIGH-ENTROPY distinctive-prefix whole-match patterns only. + // RED on bf6968f9 (glued forms return verbatim, 0 spans); GREEN after. Fixtures are BUILT from + // parts (`seg`) so no contiguous literal credential appears in SOURCE (GitHub push protection). ─── + describe("SEC-SECRET-GLUED-PREFIX-001 glued distinctive-prefix credential (leading \\b evasion)", () => { + const seg = (...p: string[]): string => p.join(""); // pragma: allowlist secret + const SGPOOL = "ABCdefGHIjkl0123456789abcdefghijkLMNopqrstuvwxyz0123456789"; // pragma: allowlist secret + const SG_SECRET = seg("SG.", SGPOOL.slice(0, 22), ".", SGPOOL.slice(0, 43)); // pragma: allowlist secret + + // Patterns whose leading `\b` was DROPPED — each is a high-signal literal prefix + LONG high-entropy + // body (body excludes separators OR the prefix is astronomically distinctive), so allowing a glued + // prefix cannot over-match a plausible benign identifier. [name, whole-secret]. + const DEBOUNDED: Array<[string, string]> = [ + ["HuggingFace hf_", seg("hf_", "AbCdEfGhIjKlMnOpQrStUvWxYz01234567")], // pragma: allowlist secret — 34 base62 + ["Groq gsk_", seg("gsk_", "abcdefghijklmnopqrstuvwxyz0123456789ABCDwx")], // pragma: allowlist secret — 42 base62 + ["npm npm_", seg("npm_", "abcdefghijklmnopqrstuvwxyz0123456789")], // pragma: allowlist secret — 36 base62 + ["DigitalOcean dop_v1_", seg("dop_v1_", "0123456789abcdef".repeat(4))], // pragma: allowlist secret — 64 hex + ["Google GOCSPX-", seg("GOCSPX-", "abcdefghijklmnop_qrstuvwx")], // pragma: allowlist secret + ["GitLab glpat-", seg("glpat-", "ABCdef0123456789ghijkLMNop")], // pragma: allowlist secret + ["SendGrid SG.<22>.<43>", SG_SECRET], // pragma: allowlist secret + ["Square sq0atp-", seg("sq0atp-", "0123456789abcdefghijklABCDwxyz")], // pragma: allowlist secret + ["Square sq0csp-", seg("sq0csp-", "0123456789abcdefghijklABCDwxyz")], // pragma: allowlist secret + ["GitHub ghp_", seg("ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")], // pragma: allowlist secret — 36 body + ["GitHub gho_", seg("gho_", "ABCDEF0123456789GHIJKL")], // pragma: allowlist secret — 22 body + ["AWS AKIA", seg("AKIA", "IOSFODNN7EXAMPLE")], // pragma: allowlist secret — 16 [0-9A-Z] + ["Slack xoxb-", seg("xoxb-", "EXAMPLENOTAREALSLACKTOKEN")], // pragma: allowlist secret + ]; + + it("RED→GREEN: a glued (word-char-prefixed) credential is redacted, benign prefix char preserved", () => { + for (const [name, secret] of DEBOUNDED) { + // Standalone still redacts (no regression on the existing whole-value behavior). + expect(redactSecretShapesInString(secret), `${name} standalone`).toBe(M); + // GLUED after a single word char, and after a multi-char word — only the credential is redacted. + expect(redactSecretShapesInString(seg("X", secret)), `${name} X-glued`).toBe(`X${M}`); + expect(redactSecretShapesInString(seg("key", secret)), `${name} key-glued`).toBe(`key${M}`); + } + }); + + it("reports the glued credential as a span that EXCLUDES the benign leading char (byte-preserving)", () => { + for (const [name, secret] of DEBOUNDED) { + const input = seg("key", secret); + const spans = findSecretShapeSpans(input); + expect(spans.length, name).toBe(1); + // The span is exactly the credential (prefix+body) — the leading `key` is outside it. + expect(input.slice(spans[0]!.start, spans[0]!.end), name).toBe(secret); + expect(spans[0]!.replacement, name).toBe(M); + // Splicing reproduces the in-place scrubber output (path parity, no off-by-one). + const spliced = input.slice(0, spans[0]!.start) + spans[0]!.replacement + input.slice(spans[0]!.end); + expect(spliced, name).toBe(redactSecretShapesInString(input)); + expect(spliced, name).toBe(`key${M}`); + } + }); + + it("still redacts through the EXISTING delimited paths (whitespace / quote / = / : / leading zero-width)", () => { + const [, HFV] = DEBOUNDED[0]!; // hf_ secret + expect(redactSecretShapesInString(`leak ${HFV} here`)).toBe(`leak ${M} here`); + expect(redactSecretShapesInString(`"${HFV}"`)).toBe(`"${M}"`); + expect(redactSecretShapesInString(`x=${HFV}`)).toBe(`x=${M}`); + expect(redactSecretShapesInString(`ref: ${HFV}`)).toBe(`ref: ${M}`); + expect(redactSecretShapesInString(`​${HFV}`)).toBe(`​${M}`); // leading zero-width + }); + + // KEPT the leading `\b` — the prefix is SHORT / low-signal and glues into a common word (or `AI`/`app` + // fragment), so a glued form would over-redact a benign identifier. Each: standalone STILL redacts, + // but the glued/word-embedded benign form is UNCHANGED. [name, standalone-secret, glued-BENIGN]. + const KEPT: Array<[string, string, string]> = [ + ["sk- (desk-/risk-/task-)", seg("sk-", "abcdefghijklmnopqrstuv0123456789"), "desk-management-framework-v2extras"], // pragma: allowlist secret + ["xai- (…xai)", seg("xai-", "abcdefghijklmnop0123456789"), seg("proxai-", "abcdefghijklmnop0123456789")], // pragma: allowlist secret + ["Stripe sk_live_ (desk_/risk_)", seg("sk", "_live_", "0123456789abcdefABCD"), seg("desk", "_live_", "0123456789abcdefABCD")], // pragma: allowlist secret + ["github_pat_ (natural phrase)", seg("github_pat_", "11ABCDE0aBcDeFgHiJkL0"), seg("my", "github_pat_", "reference0token0id00")], // pragma: allowlist secret + ["Slack xapp- (maxapp-)", seg("xapp-", "1-A0123ABCD-4567890123"), "maxapp-config-value-here"], // pragma: allowlist secret + ["Google AIza (openAIza…)", seg("AIza", "SyDabcdefghijklmnopqrstuvwxyz012345"), seg("open", "AIza", "SyDabcdefghijklmnopqrstuvwxyz012345")], // pragma: allowlist secret + ["Google ya29. (maya29.field)", seg("ya29.", "a0AfBbyDtestTokenValue0123456789ABCDEF"), "maya29.profile_image_url_field_v2"], // pragma: allowlist secret + ]; + + it("NO-DEGRADE: KEPT-boundary short/low-signal prefixes still redact standalone but NOT when glued into a word", () => { + for (const [name, standalone, gluedBenign] of KEPT) { + expect(redactSecretShapesInString(standalone), `${name} standalone`).toBe(M); + expect(redactSecretShapesInString(gluedBenign), `${name} glued-benign`).toBe(gluedBenign); + expect(findSecretShapeSpans(gluedBenign), `${name} glued-benign span`).toEqual([]); + } + }); + + // NO-DEGRADE benign-identifier corpus — MUST be returned byte-identical (zero redaction). These are + // the ordinary identifiers/blobs the shared detector protects: short/low-entropy bodies after a + // distinctive prefix, UUIDs, base64 blobs (incl. a `_`/`-`-bearing base64url one), snake_case ids, + // `AKIA` as a plain word, hex ids, file paths, and hyphen/underscore near-misses. + const BENIGN_CORPUS = [ + "myhf_variable", // hf_ + short body + "staging_key", + seg("gsk_", "count"), // gsk_count (short body) + "sq0_index", // sq0 not followed by atp-/csp- + "npm_config", // npm_ + short body with `_` + "npm_config_cache is set", + "550e8400-e29b-41d4-a716-446655440000", // a UUID + "abc123def456ghi789jkl012mno345pqr678stuv", // pure lowercase-alnum blob — no prefix reachable + "aGVsbG8_d29ybGQ-dGhpc19pc19iZW5pZ24", // pragma: allowlist secret — base64url blob w/ `_` and `-`, not a secret shape + "user_session_reference_identifier_v2", // snake_case id + "AKIA is the aws access-key id prefix", // AKIA as a plain word (no 16-char body) + "9f8e7d6c5b4a3928170695f4e3d2c1b0", // pragma: allowlist secret — 32-hex id / git blob + "/var/log/hf_service/npm_cache/output.log", // file path with hf_/npm_ short segments + "GOCSPX_notasecret_underscore", // GOCSPX_ (underscore, not the required hyphen) + "glpat_docs", // glpat_ (underscore, not the hyphen) + "sq0abc-nothing", // sq0 near-miss (neither atp nor csp) + ]; + + it("NO-DEGRADE: the benign-identifier corpus is returned byte-identical (zero over-redaction)", () => { + for (const benign of BENIGN_CORPUS) { + expect(redactSecretShapesInString(benign), benign).toBe(benign); + expect(findSecretShapeSpans(benign), benign).toEqual([]); + } + }); + }); }); diff --git a/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts b/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts index a642adf7a..28f44ca3f 100644 --- a/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts +++ b/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts @@ -207,4 +207,42 @@ describe("SEC-EVENT-REDACTION-001 — cross-sink parity (memory egress == audit expect(json).not.toContain(HF_RAW); } }); + + // ─── SEC-SECRET-GLUED-PREFIX-001: a distinctive-prefix credential GLUED directly to a preceding word + // char (`keyhf_<34>`) under a NON-sensitive key had no word boundary before the prefix, so the + // canonical detector's leading `\b` skipped it and it egressed VERBATIM through BOTH the audit sink + // and memory read-back. The fix drops the leading `\b` on the high-entropy distinctive-prefix + // patterns, so the SAME canonical detector catches the glued credential in both sinks (parity holds + // automatically). Only the credential subspan is masked — the glued benign leading char survives. + // RED on bf6968f9 (both sinks return the glued token verbatim); GREEN after. ─── + const GSK_GLUED = seg("x", "gsk_", "abcdefghijklmnopqrstuvwxyz0123456789ABCDwx"); // pragma: allowlist secret + const GLPAT_GLUED = seg("id", "glpat-", "ABCdef0123456789ghijkLMNop"); // pragma: allowlist secret + const HF_GLUED = seg("key", "hf_", HF_BODY); // pragma: allowlist secret + const GLUED_PAYLOAD = (): Record => ({ + // non-sensitive KEY NAMES so the key-name nuke does NOT fire — the glued SHAPE must be caught. + hfref: HF_GLUED, + gskref: GSK_GLUED, + glref: GLPAT_GLUED, + note: "just a note", + }); + + it("glued distinctive-prefix credentials redact identically in the AUDIT sink and MEMORY redactDeep", async () => { + const auditDetails = await auditRedactedDetails(GLUED_PAYLOAD()); + const memoryGuard = createFridayMemoryPiiGuard("redact"); + const memoryValue = memoryGuard.redactDeep(GLUED_PAYLOAD()).value as Record; + // Cross-sink equality — memory egress == audit sink for the glued credentials. + expect(memoryValue).toEqual(auditDetails); + // The credential subspan is masked, the benign glued leading char (`key`/`x`/`id`) survives. + expect(memoryValue.hfref).toBe(`key${M}`); + expect(memoryValue.gskref).toBe(`x${M}`); + expect(memoryValue.glref).toBe(`id${M}`); + expect(memoryValue.note).toBe("just a note"); + // No credential body survives in either sink's serialization. + for (const sink of [auditDetails, memoryValue]) { + const json = JSON.stringify(sink); + for (const cred of [HF_GLUED.slice(3), GSK_GLUED.slice(1), GLPAT_GLUED.slice(2)]) { + expect(json, cred).not.toContain(cred); + } + } + }); }); From 832cd7ddd9311b3b47f71c9aded229f3a9e59ede Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.8 (1M context)" Date: Sat, 18 Jul 2026 14:28:46 -0700 Subject: [PATCH 2/7] =?UTF-8?q?fix(security):=20NO-DEGRADE=20regression=20?= =?UTF-8?q?=E2=80=94=20github-classic=20glued=20branch=20used=20a=20`=5F`-?= =?UTF-8?q?inclusive=20body=20that=20over-redacted=20benign=20snake=5Fcase?= =?UTF-8?q?=20ending=20in=20ghs/gho/ghp/ghr/ghu=20before=20`=5F`=20(walkth?= =?UTF-8?q?roughs=5Fcompleted=5Fcounter=20=E2=86=92=20walkthrou[REDACTED?= =?UTF-8?q?=5FSECRET]);=20change=20the=20classic=20branch=20body=20to=20ba?= =?UTF-8?q?se62=20(`[A-Za-z0-9]`,=20the=20real=20token=20shape)=20so=20a?= =?UTF-8?q?=20snake=5Fcase=20`=5F`=20breaks=20the=20body=20=E2=80=94=20glu?= =?UTF-8?q?ed=20real=20classic=20tokens=20still=20caught,=20benign=20ident?= =?UTF-8?q?ifiers=20unchanged;=20add=20ghs=5F=20benign=20corpus=20to=20the?= =?UTF-8?q?=20detector=20+=20all=203=20sinks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01JhCDTLngxAf6MDh4Bfcd48 --- src/security/friday-secret-shape-redactor.ts | 19 +++++++++----- .../friday-event-payload-redactor.test.ts | 16 ++++++++++++ ...friday-memory-output-filter-secret.test.ts | 18 +++++++++++++ .../friday-secret-shape-redactor.test.ts | 26 +++++++++++++++++++ ...nt-redaction-001-cross-sink-parity.test.ts | 9 +++++++ 5 files changed, 81 insertions(+), 7 deletions(-) diff --git a/src/security/friday-secret-shape-redactor.ts b/src/security/friday-secret-shape-redactor.ts index 21417f738..27d2da405 100644 --- a/src/security/friday-secret-shape-redactor.ts +++ b/src/security/friday-secret-shape-redactor.ts @@ -193,14 +193,19 @@ const SECRET_CONTENT_PATTERNS: readonly SecretContentPattern[] = [ // GitHub tokens: classic `gh?_` prefixes AND fine-grained `github_pat_`. // // GLUED-PREFIX (SEC-SECRET-GLUED-PREFIX-001): the classic `gh[opsru]_` branch drops its leading `\b` - // so a token glued to a preceding word char (`keyghp_<16+>`) is still caught — no benign identifier - // segment ends in `ghp`/`gho`/`ghs`/`ghr`/`ghu` immediately before `_` (there is no such English / - // snake_case fragment), so allowing a glued prefix cannot over-match. The `github_pat_` branch KEEPS - // its leading `\b` (moved onto the branch): `github_pat` is a NATURAL English phrase ("github personal - // access token"), so a benign flat identifier `stored_github_pat_reference_token` + a `[A-Za-z0-9_]` - // body (which INCLUDES `_`) could plausibly appear in config/source — the boundary keeps it protected. + // so a token glued to a preceding word char (`keyghp_<36 base62>`) is still caught. Its body is + // BASE62 (`[A-Za-z0-9]`, NO `_`) — the shape real classic GitHub tokens actually have — and THAT is + // what makes dropping the boundary safe: like `hf_`/`gsk_`/`npm_`, a `_`-EXCLUDING body cannot span a + // snake_case tail, so a benign word ending in `ghs`/`gho`/`ghp`/`ghr`/`ghu` before `_` (e.g. + // `walkthroughs_completed_counter`, `breakthroughs_this_quarter`, `coughs_detected_v2`) breaks the + // body at the first `_` (`ghs_completed` → 9 base62 chars < 16 → no match) and stays byte-identical. + // (The prior `[A-Za-z0-9_]{16,}` body INCLUDED `_` and DID over-redact those benign identifiers — a + // real NO-DEGRADE regression, now fixed.) The `github_pat_` branch KEEPS its leading `\b` (moved onto + // the branch) AND its `_`-inclusive body: `github_pat` is a NATURAL English phrase ("github personal + // access token") and a flat identifier `stored_github_pat_reference_token` could plausibly appear in + // config/source, so the boundary keeps it protected (a fine-grained PAT's body legitimately has `_`). { - pattern: /(?:gh[opsru]_[A-Za-z0-9_]{16,}|\bgithub_pat_[A-Za-z0-9_]{16,})\b/gu, + pattern: /(?:gh[opsru]_[A-Za-z0-9]{16,}|\bgithub_pat_[A-Za-z0-9_]{16,})\b/gu, sensitiveSpan: wholeMatchSpan, }, // Provider hyphen-prefixed API keys: OpenAI `sk-` / `sk-proj-`, `rk-`, `ak-`, xAI `xai-`. The diff --git a/test/unit/api/realtime/friday-event-payload-redactor.test.ts b/test/unit/api/realtime/friday-event-payload-redactor.test.ts index 9a16fb428..36a4a0f10 100644 --- a/test/unit/api/realtime/friday-event-payload-redactor.test.ts +++ b/test/unit/api/realtime/friday-event-payload-redactor.test.ts @@ -221,6 +221,22 @@ describe("redactEventPayload — no over-redaction (NO DEGRADE)", () => { expect(out).toEqual(payload); }); + // NO-DEGRADE (SEC-SECRET-GLUED-PREFIX-001 round-2): benign snake_case words ending in `ghs`/etc before + // `_` must NOT be over-redacted on-wire — the github-classic base62 body breaks at the `_`. The + // first-round `[A-Za-z0-9_]` body corrupted these (`walkthroughs_completed_counter` → `walkthrou[REDACTED]`). + it("preserves benign `…ghs_` identifiers (github-classic base62 body, no over-redaction)", () => { + const payload = { + walk: "walkthroughs_completed_counter", + breakt: "breakthroughs_this_quarter_list", + cough: "coughs_detected_in_recording_v2", + laugh: "laughs_per_minute_counter", + note: "metric name walkthroughs_started_and_completed today", + }; + const out = redactEventPayload(payload); + expect(out).toEqual(payload); + expect(serialize(out)).not.toContain("[REDACTED]"); + }); + it("preserves a pure-digit object key (business id, \\p{Nd} exemption)", () => { const out = redactEventPayload({ "1234567890123456": "ok" }); // pragma: allowlist secret expect(Object.keys(out)).toContain("1234567890123456"); diff --git a/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts b/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts index 7cf27bccb..355f5472c 100644 --- a/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts +++ b/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts @@ -314,4 +314,22 @@ describe("FridayMemoryOutputFilter — SEC-SECRET-GLUED-PREFIX-001 glued distinc expect(out.tags).toEqual([seg("key", "hf_docs")]); expect(out.content).toBe(seg("ref key", "hf_docs")); }); + + // NO-DEGRADE (round-2): benign snake_case words ending in `ghs`/etc before `_` MUST survive through + // EVERY output-filter leg — the github-classic base62 body breaks at the `_`. The first-round + // `[A-Za-z0-9_]` body corrupted these (`walkthroughs_completed_counter` → `walkthrou[REDACTED_SECRET]`). + it("NO-DEGRADE: benign `…ghs_` identifiers survive content / metadata / tags byte-identical", () => { + const benign = ["walkthroughs_completed_counter", "breakthroughs_this_quarter_list", "coughs_detected_in_recording_v2", "laughs_per_minute_counter"]; + const out = filter.filterItem(makeItem({ + content: "metric name walkthroughs_started_and_completed today", + metadata: { walk: benign[0], breakt: benign[1], note: "keep" }, + tags: [...benign], + })); + expect(out.content).toBe("metric name walkthroughs_started_and_completed today"); + const md = out.metadata as { walk: string; breakt: string; note: string }; + expect(md.walk).toBe(benign[0]); + expect(md.breakt).toBe(benign[1]); + expect(md.note).toBe("keep"); + expect(out.tags).toEqual(benign); // no tag dropped, none rewritten + }); }); diff --git a/test/unit/security/friday-secret-shape-redactor.test.ts b/test/unit/security/friday-secret-shape-redactor.test.ts index 362d16c37..fd8581621 100644 --- a/test/unit/security/friday-secret-shape-redactor.test.ts +++ b/test/unit/security/friday-secret-shape-redactor.test.ts @@ -516,6 +516,17 @@ describe("friday-secret-shape-redactor", () => { "GOCSPX_notasecret_underscore", // GOCSPX_ (underscore, not the required hyphen) "glpat_docs", // glpat_ (underscore, not the hyphen) "sq0abc-nothing", // sq0 near-miss (neither atp nor csp) + // GitHub-classic NO-DEGRADE (round-2): benign snake_case words ENDING in ghs/gho/ghp/ghr/ghu + // before `_`. The classic branch body is BASE62 (excludes `_`), so the snake_case `_` breaks the + // body below 16 chars → no match. (These were CORRUPTED by the first-round `[A-Za-z0-9_]` body.) + "walkthroughs_completed_counter", + "breakthroughs_this_quarter_list", + "highs_and_lows_threshold_value", + "coughs_detected_in_recording_v2", + "laughs_per_minute_counter", + "troughs_index", + "sighs_and_weighs_and_doughs", // multiple ghs-ending words + "metric name walkthroughs_started_and_completed today", // free-text sentence ]; it("NO-DEGRADE: the benign-identifier corpus is returned byte-identical (zero over-redaction)", () => { @@ -524,5 +535,20 @@ describe("friday-secret-shape-redactor", () => { expect(findSecretShapeSpans(benign), benign).toEqual([]); } }); + + // GitHub-classic sensitivity boundary (round-2 regression fix): a benign snake_case word ending in + // `ghs`/etc before `_` is UNCHANGED (base62 body breaks at the `_`), while a REAL glued classic token + // (`ghs_` + a contiguous 36-base62 body, no `_`) IS redacted. RED on the first-round `[A-Za-z0-9_]` + // body corrupted the benign form; GREEN after the base62 body. + it("GitHub-classic: benign `…ghs_` survives BUT a glued real classic token IS redacted", () => { + // Benign snake_case — the `_` after `ghs` breaks the base62 body (9 chars < 16) → unchanged. + expect(redactSecretShapesInString("walkthroughs_completed_counter")).toBe("walkthroughs_completed_counter"); + expect(redactSecretShapesInString("coughs_detected today")).toBe("coughs_detected today"); + // Real glued classic token — `ghs_` + 36 contiguous base62 (no `_`) → the credential is redacted, + // the benign leading `x`/word survives (green control that the de-`\b` still catches real tokens). + const REAL = seg("ghs_", "ABCdef0123456789ghijkLMNopqrstuvWXYZ12"); // pragma: allowlist secret — 38 base62, no `_` + expect(redactSecretShapesInString(seg("x", REAL))).toBe(`x${M}`); + expect(redactSecretShapesInString(seg("key", REAL))).toBe(`key${M}`); + }); }); }); diff --git a/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts b/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts index 28f44ca3f..0e0ad880f 100644 --- a/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts +++ b/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts @@ -224,6 +224,11 @@ describe("SEC-EVENT-REDACTION-001 — cross-sink parity (memory egress == audit gskref: GSK_GLUED, glref: GLPAT_GLUED, note: "just a note", + // NO-DEGRADE (round-2): benign snake_case words ending in `ghs`/etc before `_` MUST survive in BOTH + // sinks (the github-classic base62 body breaks at the `_`) — the first-round `_`-body corrupted these. + walk: "walkthroughs_completed_counter", + breakt: "breakthroughs_this_quarter_list", + cough: "coughs_detected_in_recording_v2", }); it("glued distinctive-prefix credentials redact identically in the AUDIT sink and MEMORY redactDeep", async () => { @@ -237,6 +242,10 @@ describe("SEC-EVENT-REDACTION-001 — cross-sink parity (memory egress == audit expect(memoryValue.gskref).toBe(`x${M}`); expect(memoryValue.glref).toBe(`id${M}`); expect(memoryValue.note).toBe("just a note"); + // Benign `…ghs_` identifiers survive byte-identical in BOTH sinks (no over-redaction). + expect(memoryValue.walk).toBe("walkthroughs_completed_counter"); + expect(memoryValue.breakt).toBe("breakthroughs_this_quarter_list"); + expect(memoryValue.cough).toBe("coughs_detected_in_recording_v2"); // No credential body survives in either sink's serialization. for (const sink of [auditDetails, memoryValue]) { const json = JSON.stringify(sink); From 58d010f35a67e63363faf9010ace5c7448cd89a1 Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.8 (1M context)" Date: Sat, 18 Jul 2026 14:38:53 -0700 Subject: [PATCH 3/7] =?UTF-8?q?fix(security):=20restore=20leading=20\b=20o?= =?UTF-8?q?n=20the=20github-classic=20branch=20=E2=80=94=20the=20classic?= =?UTF-8?q?=20prefixes=20are=20common=20English=20WORD-FRAGMENTS=20(-ghs?= =?UTF-8?q?=20ends=20walkthroughs/coughs/highs;=20"github"),=20so=20a=20ba?= =?UTF-8?q?se62=20body=20alone=20still=20over-redacted=20benign=20`?= =?UTF-8?q?=5F`=20egress=20content=20(walkthrou?= =?UTF-8?q?ghs=5FcompletedThisWeek=20=E2=86=92=20walkthrou[REDACTED=5FSECR?= =?UTF-8?q?ET]);=20the=20classic=20branch=20keeps=20\b=20(delimited/standa?= =?UTF-8?q?lone/labeled=20tokens=20still=20caught,=20glued=20is=20an=20acc?= =?UTF-8?q?epted=20documented=20gap=20like=20sk-/AIza/ya29./eyJ/xapp-);=20?= =?UTF-8?q?the=20other=2010=20de-\b'd=20prefixes=20are=20not=20word-fragme?= =?UTF-8?q?nts=20and=20stay=20de-\b'd;=20contiguous-run=20benign=20corpus?= =?UTF-8?q?=20added=20to=20detector=20+=20all=203=20sinks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01JhCDTLngxAf6MDh4Bfcd48 --- src/security/friday-secret-shape-redactor.ts | 28 ++++++----- .../friday-event-payload-redactor.test.ts | 3 +- ...friday-memory-output-filter-secret.test.ts | 2 +- .../friday-secret-shape-redactor.test.ts | 47 ++++++++++++------- ...nt-redaction-001-cross-sink-parity.test.ts | 13 +++-- 5 files changed, 56 insertions(+), 37 deletions(-) diff --git a/src/security/friday-secret-shape-redactor.ts b/src/security/friday-secret-shape-redactor.ts index 27d2da405..9f237d8f8 100644 --- a/src/security/friday-secret-shape-redactor.ts +++ b/src/security/friday-secret-shape-redactor.ts @@ -192,20 +192,22 @@ const SECRET_CONTENT_PATTERNS: readonly SecretContentPattern[] = [ }, // GitHub tokens: classic `gh?_` prefixes AND fine-grained `github_pat_`. // - // GLUED-PREFIX (SEC-SECRET-GLUED-PREFIX-001): the classic `gh[opsru]_` branch drops its leading `\b` - // so a token glued to a preceding word char (`keyghp_<36 base62>`) is still caught. Its body is - // BASE62 (`[A-Za-z0-9]`, NO `_`) — the shape real classic GitHub tokens actually have — and THAT is - // what makes dropping the boundary safe: like `hf_`/`gsk_`/`npm_`, a `_`-EXCLUDING body cannot span a - // snake_case tail, so a benign word ending in `ghs`/`gho`/`ghp`/`ghr`/`ghu` before `_` (e.g. - // `walkthroughs_completed_counter`, `breakthroughs_this_quarter`, `coughs_detected_v2`) breaks the - // body at the first `_` (`ghs_completed` → 9 base62 chars < 16 → no match) and stays byte-identical. - // (The prior `[A-Za-z0-9_]{16,}` body INCLUDED `_` and DID over-redact those benign identifiers — a - // real NO-DEGRADE regression, now fixed.) The `github_pat_` branch KEEPS its leading `\b` (moved onto - // the branch) AND its `_`-inclusive body: `github_pat` is a NATURAL English phrase ("github personal - // access token") and a flat identifier `stored_github_pat_reference_token` could plausibly appear in - // config/source, so the boundary keeps it protected (a fine-grained PAT's body legitimately has `_`). + // GLUED-PREFIX (SEC-SECRET-GLUED-PREFIX-001): the GitHub branch KEEPS its leading `\b` — UNLIKE the 10 + // other de-`\b`'d shapes, the classic prefixes are common ENGLISH WORD-FRAGMENTS: `ghs` ends + // walkthrou-GHS, breakthrou-GHS, cou-GHS, hi-GHS, lau-GHS, rou-GHS, wei-GHS, plou-GHS, borou-GHS; and + // `github_pat_` contains the word "github". So dropping the boundary would over-redact benign + // `_<16+ CONTIGUOUS base62 run>` content this SHARED egress detector processes — + // even with a base62 body: `walkthroughs_completedThisWeek` → `ghs_completedThisWeek` (17 base62) or + // `coughs_e3b0c44298fc1c14` (a content-hash suffix) would be corrupted. Because the collision surface + // is a word FRAGMENT (not a distinctive token prefix), `\b` is REQUIRED. Effect: a DELIMITED / + // standalone / labeled classic token (`ghp_`, `token: ghp_…`, `apikey=ghp_…`) is STILL caught + // (the common case); only a classic token glued DIRECTLY after a word char (`keyghp_…`) is an + // ACCEPTED, documented gap — exactly like the kept-`\b` `sk-`/`AIza`/`ya29.`/`eyJ`/`xapp-` prefixes. + // The classic body is still BASE62 (`[A-Za-z0-9]`, real-token shape) so a standalone `ghs_a_b_c` + // snake_case identifier breaks at the first `_`; the `github_pat_` branch keeps its `_`-inclusive body + // (a fine-grained PAT's body legitimately has `_`). { - pattern: /(?:gh[opsru]_[A-Za-z0-9]{16,}|\bgithub_pat_[A-Za-z0-9_]{16,})\b/gu, + pattern: /\b(?:gh[opsru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{16,})\b/gu, sensitiveSpan: wholeMatchSpan, }, // Provider hyphen-prefixed API keys: OpenAI `sk-` / `sk-proj-`, `rk-`, `ak-`, xAI `xai-`. The diff --git a/test/unit/api/realtime/friday-event-payload-redactor.test.ts b/test/unit/api/realtime/friday-event-payload-redactor.test.ts index 36a4a0f10..e5e775943 100644 --- a/test/unit/api/realtime/friday-event-payload-redactor.test.ts +++ b/test/unit/api/realtime/friday-event-payload-redactor.test.ts @@ -227,8 +227,9 @@ describe("redactEventPayload — no over-redaction (NO DEGRADE)", () => { it("preserves benign `…ghs_` identifiers (github-classic base62 body, no over-redaction)", () => { const payload = { walk: "walkthroughs_completed_counter", + contig: "walkthroughs_completedThisWeek", // CONTIGUOUS 17-base62 run — only the leading `\b` closes this breakt: "breakthroughs_this_quarter_list", - cough: "coughs_detected_in_recording_v2", + cough: "coughs_e3b0c44298fc1c14", // ghs_ + content-hash suffix laugh: "laughs_per_minute_counter", note: "metric name walkthroughs_started_and_completed today", }; diff --git a/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts b/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts index 355f5472c..013825cb7 100644 --- a/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts +++ b/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts @@ -319,7 +319,7 @@ describe("FridayMemoryOutputFilter — SEC-SECRET-GLUED-PREFIX-001 glued distinc // EVERY output-filter leg — the github-classic base62 body breaks at the `_`. The first-round // `[A-Za-z0-9_]` body corrupted these (`walkthroughs_completed_counter` → `walkthrou[REDACTED_SECRET]`). it("NO-DEGRADE: benign `…ghs_` identifiers survive content / metadata / tags byte-identical", () => { - const benign = ["walkthroughs_completed_counter", "breakthroughs_this_quarter_list", "coughs_detected_in_recording_v2", "laughs_per_minute_counter"]; + const benign = ["walkthroughs_completed_counter", "walkthroughs_completedThisWeek", "coughs_e3b0c44298fc1c14", "breakthroughs_this_quarter_list", "laughs_per_minute_counter"]; const out = filter.filterItem(makeItem({ content: "metric name walkthroughs_started_and_completed today", metadata: { walk: benign[0], breakt: benign[1], note: "keep" }, diff --git a/test/unit/security/friday-secret-shape-redactor.test.ts b/test/unit/security/friday-secret-shape-redactor.test.ts index fd8581621..1e02a19c2 100644 --- a/test/unit/security/friday-secret-shape-redactor.test.ts +++ b/test/unit/security/friday-secret-shape-redactor.test.ts @@ -434,8 +434,8 @@ describe("friday-secret-shape-redactor", () => { ["SendGrid SG.<22>.<43>", SG_SECRET], // pragma: allowlist secret ["Square sq0atp-", seg("sq0atp-", "0123456789abcdefghijklABCDwxyz")], // pragma: allowlist secret ["Square sq0csp-", seg("sq0csp-", "0123456789abcdefghijklABCDwxyz")], // pragma: allowlist secret - ["GitHub ghp_", seg("ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")], // pragma: allowlist secret — 36 body - ["GitHub gho_", seg("gho_", "ABCDEF0123456789GHIJKL")], // pragma: allowlist secret — 22 body + // NB: GitHub classic `gh[opsru]_` / `github_pat_` are NOT here — their prefixes are common English + // word-fragments (`-ghs`, "github"), so they KEEP `\b` (glued case is an accepted gap); see KEPT. ["AWS AKIA", seg("AKIA", "IOSFODNN7EXAMPLE")], // pragma: allowlist secret — 16 [0-9A-Z] ["Slack xoxb-", seg("xoxb-", "EXAMPLENOTAREALSLACKTOKEN")], // pragma: allowlist secret ]; @@ -474,14 +474,17 @@ describe("friday-secret-shape-redactor", () => { expect(redactSecretShapesInString(`​${HFV}`)).toBe(`​${M}`); // leading zero-width }); - // KEPT the leading `\b` — the prefix is SHORT / low-signal and glues into a common word (or `AI`/`app` - // fragment), so a glued form would over-redact a benign identifier. Each: standalone STILL redacts, - // but the glued/word-embedded benign form is UNCHANGED. [name, standalone-secret, glued-BENIGN]. + // KEPT the leading `\b` — the prefix is SHORT / low-signal and glues into a common word, or is itself + // a common English WORD-FRAGMENT (`AI`/`app`/`-ghs`/"github"), so a glued form would over-redact a + // benign identifier. Each: standalone STILL redacts, but the glued/word-embedded benign form is + // UNCHANGED. [name, standalone-secret, glued-BENIGN]. const KEPT: Array<[string, string, string]> = [ ["sk- (desk-/risk-/task-)", seg("sk-", "abcdefghijklmnopqrstuv0123456789"), "desk-management-framework-v2extras"], // pragma: allowlist secret ["xai- (…xai)", seg("xai-", "abcdefghijklmnop0123456789"), seg("proxai-", "abcdefghijklmnop0123456789")], // pragma: allowlist secret ["Stripe sk_live_ (desk_/risk_)", seg("sk", "_live_", "0123456789abcdefABCD"), seg("desk", "_live_", "0123456789abcdefABCD")], // pragma: allowlist secret ["github_pat_ (natural phrase)", seg("github_pat_", "11ABCDE0aBcDeFgHiJkL0"), seg("my", "github_pat_", "reference0token0id00")], // pragma: allowlist secret + // GitHub classic — `-ghs`/`-gho`/… are word-endings (walkthroughs), so a glued form is NOT caught. + ["GitHub classic ghp_ (…ghs word-ending)", seg("ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"), seg("walkthroughs_", "completedThisWeekX")], // pragma: allowlist secret ["Slack xapp- (maxapp-)", seg("xapp-", "1-A0123ABCD-4567890123"), "maxapp-config-value-here"], // pragma: allowlist secret ["Google AIza (openAIza…)", seg("AIza", "SyDabcdefghijklmnopqrstuvwxyz012345"), seg("open", "AIza", "SyDabcdefghijklmnopqrstuvwxyz012345")], // pragma: allowlist secret ["Google ya29. (maya29.field)", seg("ya29.", "a0AfBbyDtestTokenValue0123456789ABCDEF"), "maya29.profile_image_url_field_v2"], // pragma: allowlist secret @@ -520,6 +523,9 @@ describe("friday-secret-shape-redactor", () => { // before `_`. The classic branch body is BASE62 (excludes `_`), so the snake_case `_` breaks the // body below 16 chars → no match. (These were CORRUPTED by the first-round `[A-Za-z0-9_]` body.) "walkthroughs_completed_counter", + "walkthroughs_completedThisWeek", // CONTIGUOUS 17-base62 run after ghs_ — only the leading `\b` closes this + "coughs_e3b0c44298fc1c14", // ghs_ + a content-hash suffix (contiguous base62) + "highs_thresholdValueConfig1", "breakthroughs_this_quarter_list", "highs_and_lows_threshold_value", "coughs_detected_in_recording_v2", @@ -536,19 +542,26 @@ describe("friday-secret-shape-redactor", () => { } }); - // GitHub-classic sensitivity boundary (round-2 regression fix): a benign snake_case word ending in - // `ghs`/etc before `_` is UNCHANGED (base62 body breaks at the `_`), while a REAL glued classic token - // (`ghs_` + a contiguous 36-base62 body, no `_`) IS redacted. RED on the first-round `[A-Za-z0-9_]` - // body corrupted the benign form; GREEN after the base62 body. - it("GitHub-classic: benign `…ghs_` survives BUT a glued real classic token IS redacted", () => { - // Benign snake_case — the `_` after `ghs` breaks the base62 body (9 chars < 16) → unchanged. + // GitHub-classic sensitivity boundary (round-3): the classic prefixes are common English + // WORD-FRAGMENTS (`-ghs` ends walkthroughs/coughs/highs; "github" is a word), so — unlike the 10 + // other de-`\b`'d shapes — the classic branch KEEPS `\b`. A benign `<-ghs word>_` is UNCHANGED (the boundary blocks it, even for a contiguous run that a base62 body alone did + // not close). A DELIMITED / standalone / labeled classic token IS still caught (the common case); a + // classic token glued DIRECTLY after a word char is an ACCEPTED, documented gap (like sk-/AIza/ya29.). + it("GitHub-classic: benign `…ghs_` survives; DELIMITED/standalone token redacts; GLUED is an accepted gap", () => { + // Benign — the leading `\b` blocks the match; UNCHANGED even for a CONTIGUOUS 17+ base62 run. expect(redactSecretShapesInString("walkthroughs_completed_counter")).toBe("walkthroughs_completed_counter"); - expect(redactSecretShapesInString("coughs_detected today")).toBe("coughs_detected today"); - // Real glued classic token — `ghs_` + 36 contiguous base62 (no `_`) → the credential is redacted, - // the benign leading `x`/word survives (green control that the de-`\b` still catches real tokens). - const REAL = seg("ghs_", "ABCdef0123456789ghijkLMNopqrstuvWXYZ12"); // pragma: allowlist secret — 38 base62, no `_` - expect(redactSecretShapesInString(seg("x", REAL))).toBe(`x${M}`); - expect(redactSecretShapesInString(seg("key", REAL))).toBe(`key${M}`); + expect(redactSecretShapesInString("walkthroughs_completedThisWeek")).toBe("walkthroughs_completedThisWeek"); // 17 contiguous base62 + expect(redactSecretShapesInString("coughs_e3b0c44298fc1c14")).toBe("coughs_e3b0c44298fc1c14"); // hash suffix + expect(redactSecretShapesInString("highs_thresholdValueConfig1")).toBe("highs_thresholdValueConfig1"); + const TOK = seg("ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"); // pragma: allowlist secret — 36 base62 + // DELIMITED / standalone / labeled classic token IS still caught (whitespace, `=`, bare value). + expect(redactSecretShapesInString(TOK)).toBe(M); + expect(redactSecretShapesInString(`token ${TOK} used`)).toBe(`token ${M} used`); + expect(redactSecretShapesInString(`apikey=${TOK}`)).toBe(`apikey=${M}`); + // GLUED directly after a word char is an ACCEPTED gap for these word-fragment prefixes. + expect(redactSecretShapesInString(seg("key", TOK))).toBe(seg("key", TOK)); + expect(findSecretShapeSpans(seg("key", TOK))).toEqual([]); }); }); }); diff --git a/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts b/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts index 0e0ad880f..b042ab0be 100644 --- a/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts +++ b/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts @@ -224,11 +224,12 @@ describe("SEC-EVENT-REDACTION-001 — cross-sink parity (memory egress == audit gskref: GSK_GLUED, glref: GLPAT_GLUED, note: "just a note", - // NO-DEGRADE (round-2): benign snake_case words ending in `ghs`/etc before `_` MUST survive in BOTH - // sinks (the github-classic base62 body breaks at the `_`) — the first-round `_`-body corrupted these. + // NO-DEGRADE: benign words ending in `ghs`/etc before `_` MUST survive in BOTH sinks — the + // github-classic branch keeps `\b`, so a `<-ghs word>_` is not over-matched. walk: "walkthroughs_completed_counter", + contig: "walkthroughs_completedThisWeek", // CONTIGUOUS 17-base62 run — only the leading `\b` closes this breakt: "breakthroughs_this_quarter_list", - cough: "coughs_detected_in_recording_v2", + cough: "coughs_e3b0c44298fc1c14", // ghs_ + content-hash suffix }); it("glued distinctive-prefix credentials redact identically in the AUDIT sink and MEMORY redactDeep", async () => { @@ -242,10 +243,12 @@ describe("SEC-EVENT-REDACTION-001 — cross-sink parity (memory egress == audit expect(memoryValue.gskref).toBe(`x${M}`); expect(memoryValue.glref).toBe(`id${M}`); expect(memoryValue.note).toBe("just a note"); - // Benign `…ghs_` identifiers survive byte-identical in BOTH sinks (no over-redaction). + // Benign `…ghs_` identifiers survive byte-identical in BOTH sinks (no over-redaction), including + // a CONTIGUOUS base62 run that only the leading `\b` closes. expect(memoryValue.walk).toBe("walkthroughs_completed_counter"); + expect(memoryValue.contig).toBe("walkthroughs_completedThisWeek"); expect(memoryValue.breakt).toBe("breakthroughs_this_quarter_list"); - expect(memoryValue.cough).toBe("coughs_detected_in_recording_v2"); + expect(memoryValue.cough).toBe("coughs_e3b0c44298fc1c14"); // No credential body survives in either sink's serialization. for (const sink of [auditDetails, memoryValue]) { const json = JSON.stringify(sink); From 9a7311de477af6d5d98d319735f73978f3862b7c Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.8 (1M context)" Date: Sat, 18 Jul 2026 15:08:26 -0700 Subject: [PATCH 4/7] =?UTF-8?q?fix(security):=20restore=20leading=20\b=20o?= =?UTF-8?q?n=20the=20AWS=20AKIA=20branch=20=E2=80=94=20its=20alternation?= =?UTF-8?q?=20embeds=20English-word=20/=20base32-constructible=20fragments?= =?UTF-8?q?=20(ASIA/AIDA/AGPA)=20and=20its=20body=20is=20[0-9A-Z]{16}=20(t?= =?UTF-8?q?he=20all-caps-constant=20/=20ULID=20alphabet),=20so=20de-\b=20o?= =?UTF-8?q?ver-redacted=20benign=20all-caps/ULID=20ids=20(012345AGPABCDEFG?= =?UTF-8?q?HJKMNPQRST=20=E2=86=92=20012345[REDACTED=5FSECRET];=20AUSTRALAS?= =?UTF-8?q?IAWIDE=E2=80=A6=20=E2=86=92=20AUSTRAL[REDACTED=5FSECRET]);=20th?= =?UTF-8?q?e=20AKIA=20branch=20keeps=20\b=20(delimited/standalone/labeled?= =?UTF-8?q?=20AWS=20keys=20still=20caught,=20glued=20is=20an=20accepted=20?= =?UTF-8?q?documented=20gap=20like=20classic-github/sk-/AIza/ya29./xapp-);?= =?UTF-8?q?=20the=20other=209=20de-\b'd=20prefixes=20(hf=5F/gsk=5F/npm=5F/?= =?UTF-8?q?dop=5Fv1=5F/GOCSPX-/glpat-/SG./sq0/xox)=20empirically=20re-conf?= =?UTF-8?q?irmed=20collision-free=20(non-word-fragment=20vendor=20prefixes?= =?UTF-8?q?,=200=20diffs=20vs=20base=20on=209k=20natural-benign=20inputs);?= =?UTF-8?q?=20ULID/all-caps=20benign=20corpus=20added=20to=20detector=20+?= =?UTF-8?q?=20all=203=20sinks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01JhCDTLngxAf6MDh4Bfcd48 --- src/security/friday-secret-shape-redactor.ts | 18 ++++++--- .../friday-event-payload-redactor.test.ts | 3 ++ ...friday-memory-output-filter-secret.test.ts | 4 +- .../friday-secret-shape-redactor.test.ts | 39 +++++++++++++++++-- ...nt-redaction-001-cross-sink-parity.test.ts | 6 +++ 5 files changed, 61 insertions(+), 9 deletions(-) diff --git a/src/security/friday-secret-shape-redactor.ts b/src/security/friday-secret-shape-redactor.ts index 9f237d8f8..0872d27a4 100644 --- a/src/security/friday-secret-shape-redactor.ts +++ b/src/security/friday-secret-shape-redactor.ts @@ -241,12 +241,20 @@ const SECRET_CONTENT_PATTERNS: readonly SecretContentPattern[] = [ // AWS access-key id (`AKIA…` long-term + STS/temporary variants). NB: the AWS SECRET access key is // a shapeless 40-char base64 with no prefix — it is caught by its KEY NAME (`secretAccessKey`, // in the sensitive-field-name set), NOT a shape, so it is not (and must not be) a content pattern. - // GLUED-PREFIX: leading `\b` dropped (see hf_ note). The `A(?:KIA|SIA|…)` prefix + 16 CONTIGUOUS - // `[0-9A-Z]` (uppercase/digit only — a `_`, `-`, or lowercase char breaks the body) is the gate: a - // benign snake_case constant like `REGION_ASIA_PACIFIC_1` cannot span it (the `_` after `ASIA` ends - // the body at 0 chars), so allowing a glued prefix (`xAKIA<16>`) does not over-match plausible ids. + // GLUED-PREFIX: leading `\b` KEPT deliberately (UNLIKE the base62 vendor prefixes). The `A(?:KIA|SIA|…)` + // alternation embeds ENGLISH-WORD / base32-constructible fragments — `ASIA` (Asia / Eurasia / + // Australasia), `AIDA`, and `AGPA`/`ANPA`/`ANVA` (constructible from Crockford base32) — AND its body is + // `[0-9A-Z]{16}`, i.e. UPPERCASE + DIGIT, exactly the alphabet of an all-caps constant or a ULID. So + // de-`\b` over-redacts benign all-caps / ULID / base32 identifiers this SHARED egress detector + // processes: a 26-char ULID `012345AGPABCDEFGHJKMNPQRST` → `012345[REDACTED]`, `AUSTRALASIAWIDE…` → + // `AUSTRAL[REDACTED]`, a base32 OTP `ASIAMFRGGZDFMZTWQ2LK`. (The earlier "REGION_ASIA_PACIFIC_1 cannot + // span it" reasoning was WRONG — it only considered `_`-separated forms and missed all-caps-contiguous + // / ULID / base32.) Same word-fragment class we keep `\b` on for classic-github (`-ghs`), `xapp` (app), + // `sk-`, `AIza` (AI), `ya29`. Effect: standalone / delimited / labeled AWS keys (`AKIA<16>`, + // `token AKIA…`, `apikey=AKIA…` — AWS keys are ~always delimited in env/config) are STILL caught; only + // a key glued DIRECTLY after a word char (`xAKIA<16>`) is an ACCEPTED, documented gap. { - pattern: /A(?:KIA|SIA|GPA|IDA|ROA|IPA|NPA|NVA)[0-9A-Z]{16}\b/gu, + pattern: /\bA(?:KIA|SIA|GPA|IDA|ROA|IPA|NPA|NVA)[0-9A-Z]{16}\b/gu, sensitiveSpan: wholeMatchSpan, }, // Google API key — `AIza` + 35 chars of `[0-9A-Za-z_-]` (documented 39-char format). diff --git a/test/unit/api/realtime/friday-event-payload-redactor.test.ts b/test/unit/api/realtime/friday-event-payload-redactor.test.ts index e5e775943..61e81922a 100644 --- a/test/unit/api/realtime/friday-event-payload-redactor.test.ts +++ b/test/unit/api/realtime/friday-event-payload-redactor.test.ts @@ -232,6 +232,9 @@ describe("redactEventPayload — no over-redaction (NO DEGRADE)", () => { cough: "coughs_e3b0c44298fc1c14", // ghs_ + content-hash suffix laugh: "laughs_per_minute_counter", note: "metric name walkthroughs_started_and_completed today", + // AWS AKIA word-fragment: benign ULID / all-caps id with ASIA/AGPA glued after a word char. + ulid: "012345AGPABCDEFGHJKMNPQRST", + caps: "AUSTRALASIAWIDEDEPLOYMENT01", }; const out = redactEventPayload(payload); expect(out).toEqual(payload); diff --git a/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts b/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts index 013825cb7..60c7c6aa1 100644 --- a/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts +++ b/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts @@ -319,7 +319,9 @@ describe("FridayMemoryOutputFilter — SEC-SECRET-GLUED-PREFIX-001 glued distinc // EVERY output-filter leg — the github-classic base62 body breaks at the `_`. The first-round // `[A-Za-z0-9_]` body corrupted these (`walkthroughs_completed_counter` → `walkthrou[REDACTED_SECRET]`). it("NO-DEGRADE: benign `…ghs_` identifiers survive content / metadata / tags byte-identical", () => { - const benign = ["walkthroughs_completed_counter", "walkthroughs_completedThisWeek", "coughs_e3b0c44298fc1c14", "breakthroughs_this_quarter_list", "laughs_per_minute_counter"]; + const benign = ["walkthroughs_completed_counter", "walkthroughs_completedThisWeek", "coughs_e3b0c44298fc1c14", "breakthroughs_this_quarter_list", "laughs_per_minute_counter", + // AWS AKIA word-fragment: benign ULID / all-caps id with ASIA/AGPA glued after a word char. + "012345AGPABCDEFGHJKMNPQRST", "AUSTRALASIAWIDEDEPLOYMENT01"]; const out = filter.filterItem(makeItem({ content: "metric name walkthroughs_started_and_completed today", metadata: { walk: benign[0], breakt: benign[1], note: "keep" }, diff --git a/test/unit/security/friday-secret-shape-redactor.test.ts b/test/unit/security/friday-secret-shape-redactor.test.ts index 1e02a19c2..fe041215a 100644 --- a/test/unit/security/friday-secret-shape-redactor.test.ts +++ b/test/unit/security/friday-secret-shape-redactor.test.ts @@ -434,9 +434,10 @@ describe("friday-secret-shape-redactor", () => { ["SendGrid SG.<22>.<43>", SG_SECRET], // pragma: allowlist secret ["Square sq0atp-", seg("sq0atp-", "0123456789abcdefghijklABCDwxyz")], // pragma: allowlist secret ["Square sq0csp-", seg("sq0csp-", "0123456789abcdefghijklABCDwxyz")], // pragma: allowlist secret - // NB: GitHub classic `gh[opsru]_` / `github_pat_` are NOT here — their prefixes are common English - // word-fragments (`-ghs`, "github"), so they KEEP `\b` (glued case is an accepted gap); see KEPT. - ["AWS AKIA", seg("AKIA", "IOSFODNN7EXAMPLE")], // pragma: allowlist secret — 16 [0-9A-Z] + // NB: GitHub classic `gh[opsru]_` / `github_pat_` and AWS `AKIA…` are NOT here — their prefixes are + // English-word / base32 fragments (`-ghs`, "github", `ASIA`/`AIDA`), so they KEEP `\b` (glued case is + // an accepted gap); see KEPT. The 9 shapes above are non-word-fragment vendor prefixes with a body + // that excludes `_`/`-` (or an ultra-distinctive prefix), so they safely stay de-`\b`'d. ["Slack xoxb-", seg("xoxb-", "EXAMPLENOTAREALSLACKTOKEN")], // pragma: allowlist secret ]; @@ -485,6 +486,9 @@ describe("friday-secret-shape-redactor", () => { ["github_pat_ (natural phrase)", seg("github_pat_", "11ABCDE0aBcDeFgHiJkL0"), seg("my", "github_pat_", "reference0token0id00")], // pragma: allowlist secret // GitHub classic — `-ghs`/`-gho`/… are word-endings (walkthroughs), so a glued form is NOT caught. ["GitHub classic ghp_ (…ghs word-ending)", seg("ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"), seg("walkthroughs_", "completedThisWeekX")], // pragma: allowlist secret + // AWS AKIA — `ASIA`/`AIDA` are English words + base32-constructible, body is all-caps/digit (ULID + // alphabet), so a glued form (a 26-char ULID `012345AGPA…`) is NOT caught; standalone still is. + ["AWS AKIA (ASIA/AIDA word-fragment)", seg("AKIA", "IOSFODNN7EXAMPLE"), "012345AGPABCDEFGHJKMNPQRST"], // pragma: allowlist secret — glued-benign is a ULID ["Slack xapp- (maxapp-)", seg("xapp-", "1-A0123ABCD-4567890123"), "maxapp-config-value-here"], // pragma: allowlist secret ["Google AIza (openAIza…)", seg("AIza", "SyDabcdefghijklmnopqrstuvwxyz012345"), seg("open", "AIza", "SyDabcdefghijklmnopqrstuvwxyz012345")], // pragma: allowlist secret ["Google ya29. (maya29.field)", seg("ya29.", "a0AfBbyDtestTokenValue0123456789ABCDEF"), "maya29.profile_image_url_field_v2"], // pragma: allowlist secret @@ -514,6 +518,14 @@ describe("friday-secret-shape-redactor", () => { "aGVsbG8_d29ybGQ-dGhpc19pc19iZW5pZ24", // pragma: allowlist secret — base64url blob w/ `_` and `-`, not a secret shape "user_session_reference_identifier_v2", // snake_case id "AKIA is the aws access-key id prefix", // AKIA as a plain word (no 16-char body) + // AWS AKIA NO-DEGRADE: benign all-caps / ULID / base32 ids where an ASIA/AIDA/AGPA fragment is GLUED + // after a word char — the AKIA branch keeps `\b`, so these are UNCHANGED (they were CORRUPTED while + // AKIA was de-`\b`'d). (A DELIMITED `"ASIA<16>"` — quote boundary — is caught by BASE too, so it is + // NOT a benign-unchanged case; only the GLUED forms are the fix's concern.) + "012345AGPABCDEFGHJKMNPQRST", // 26-char Crockford-base32 ULID (AGPA glued after digit `5`) + "AUSTRALASIAWIDEDEPLOYMENT01", // all-caps constant, ASIA glued after `L` + "EURASIAREGIONCODE0123456789", // all-caps, ASIA glued after `R` + "PROJECTAIDABUILDPIPELINE42X", // all-caps, AIDA glued after `T` "9f8e7d6c5b4a3928170695f4e3d2c1b0", // pragma: allowlist secret — 32-hex id / git blob "/var/log/hf_service/npm_cache/output.log", // file path with hf_/npm_ short segments "GOCSPX_notasecret_underscore", // GOCSPX_ (underscore, not the required hyphen) @@ -563,5 +575,26 @@ describe("friday-secret-shape-redactor", () => { expect(redactSecretShapesInString(seg("key", TOK))).toBe(seg("key", TOK)); expect(findSecretShapeSpans(seg("key", TOK))).toEqual([]); }); + + // AWS AKIA sensitivity boundary (round-4): the alternation embeds English-word / base32-constructible + // fragments (`ASIA`/`AIDA`/`AGPA`) and its body is `[0-9A-Z]{16}` (the ULID / all-caps-constant + // alphabet), so — like classic-github — the AKIA branch KEEPS `\b`. A benign all-caps / ULID / base32 + // id where the fragment is GLUED after a word char is UNCHANGED. A DELIMITED / standalone / labeled + // AWS key IS still caught (the common case — AWS keys are ~always delimited in env/config); a key + // glued directly after a word char (`xAKIA<16>`) is an ACCEPTED, documented gap. + it("AWS AKIA: benign glued ULID/all-caps survives; DELIMITED/standalone key redacts; GLUED is an accepted gap", () => { + // Benign all-caps / ULID with a fragment GLUED after a word char → UNCHANGED (the `\b` blocks it). + expect(redactSecretShapesInString("012345AGPABCDEFGHJKMNPQRST")).toBe("012345AGPABCDEFGHJKMNPQRST"); // ULID + expect(redactSecretShapesInString("AUSTRALASIAWIDEDEPLOYMENT01")).toBe("AUSTRALASIAWIDEDEPLOYMENT01"); + expect(redactSecretShapesInString("PROJECTAIDABUILDPIPELINE42X")).toBe("PROJECTAIDABUILDPIPELINE42X"); + const KEY = seg("AKIA", "IOSFODNN7EXAMPLE"); // pragma: allowlist secret — AKIA + 16 [0-9A-Z] + // DELIMITED / standalone / labeled AWS key IS still caught (whitespace, `=`, bare value). + expect(redactSecretShapesInString(KEY)).toBe(M); + expect(redactSecretShapesInString(`token ${KEY} used`)).toBe(`token ${M} used`); + expect(redactSecretShapesInString(`apikey=${KEY}`)).toBe(`apikey=${M}`); + // GLUED directly after a word char is an ACCEPTED gap for this word-fragment prefix. + expect(redactSecretShapesInString(seg("x", KEY))).toBe(seg("x", KEY)); + expect(findSecretShapeSpans(seg("x", KEY))).toEqual([]); + }); }); }); diff --git a/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts b/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts index b042ab0be..a4fa0890a 100644 --- a/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts +++ b/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts @@ -230,6 +230,10 @@ describe("SEC-EVENT-REDACTION-001 — cross-sink parity (memory egress == audit contig: "walkthroughs_completedThisWeek", // CONTIGUOUS 17-base62 run — only the leading `\b` closes this breakt: "breakthroughs_this_quarter_list", cough: "coughs_e3b0c44298fc1c14", // ghs_ + content-hash suffix + // AWS AKIA word-fragment: a benign ULID / all-caps id with ASIA/AGPA glued after a word char MUST + // survive in BOTH sinks — the AKIA branch keeps `\b` (was corrupted while AKIA was de-`\b`'d). + ulid: "012345AGPABCDEFGHJKMNPQRST", + caps: "AUSTRALASIAWIDEDEPLOYMENT01", }); it("glued distinctive-prefix credentials redact identically in the AUDIT sink and MEMORY redactDeep", async () => { @@ -249,6 +253,8 @@ describe("SEC-EVENT-REDACTION-001 — cross-sink parity (memory egress == audit expect(memoryValue.contig).toBe("walkthroughs_completedThisWeek"); expect(memoryValue.breakt).toBe("breakthroughs_this_quarter_list"); expect(memoryValue.cough).toBe("coughs_e3b0c44298fc1c14"); + expect(memoryValue.ulid).toBe("012345AGPABCDEFGHJKMNPQRST"); + expect(memoryValue.caps).toBe("AUSTRALASIAWIDEDEPLOYMENT01"); // No credential body survives in either sink's serialization. for (const sink of [auditDetails, memoryValue]) { const json = JSON.stringify(sink); From 7021926c86f3bde61517b6fbf955fc8b24045829 Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.8 (1M context)" Date: Sat, 18 Jul 2026 15:43:29 -0700 Subject: [PATCH 5/7] =?UTF-8?q?chore(security):=20satisfy=20CI=20detect-se?= =?UTF-8?q?crets-hook=20on=20the=20benign=20AKIA-family=20NO-DEGRADE=20fix?= =?UTF-8?q?tures=20=E2=80=94=20add=20narrow=20inline=20`pragma:=20allowlis?= =?UTF-8?q?t=20secret`=20markers=20to=20each=20benign=20ULID/all-caps/base?= =?UTF-8?q?32/high-entropy=20test=20literal=20the=20scanner=20misclassifie?= =?UTF-8?q?s=20(AWS=20Access=20Key=20/=20Base64=20High=20Entropy=20false?= =?UTF-8?q?=20positives),=20and=20truncate/obfuscate=20the=20illustrative?= =?UTF-8?q?=20secret-shaped=20examples=20in=20the=20AKIA=20source=20commen?= =?UTF-8?q?t;=20NO=20change=20to=20.secrets.baseline=20or=20the=20exclude-?= =?UTF-8?q?files=20glob=20(the=20scanner's=20own=20suggested=20mitigation)?= =?UTF-8?q?.=20Local=20`detect-secrets-hook`=20now=20reports=20no=20new=20?= =?UTF-8?q?secrets;=20logic=20unchanged=20(pragmas=20are=20comments)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01JhCDTLngxAf6MDh4Bfcd48 --- src/security/friday-secret-shape-redactor.ts | 4 ++-- .../friday-event-payload-redactor.test.ts | 4 ++-- .../friday-memory-output-filter-secret.test.ts | 2 +- .../friday-secret-shape-redactor.test.ts | 16 ++++++++-------- ...event-redaction-001-cross-sink-parity.test.ts | 8 ++++---- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/security/friday-secret-shape-redactor.ts b/src/security/friday-secret-shape-redactor.ts index 0872d27a4..9641ef1b9 100644 --- a/src/security/friday-secret-shape-redactor.ts +++ b/src/security/friday-secret-shape-redactor.ts @@ -246,8 +246,8 @@ const SECRET_CONTENT_PATTERNS: readonly SecretContentPattern[] = [ // Australasia), `AIDA`, and `AGPA`/`ANPA`/`ANVA` (constructible from Crockford base32) — AND its body is // `[0-9A-Z]{16}`, i.e. UPPERCASE + DIGIT, exactly the alphabet of an all-caps constant or a ULID. So // de-`\b` over-redacts benign all-caps / ULID / base32 identifiers this SHARED egress detector - // processes: a 26-char ULID `012345AGPABCDEFGHJKMNPQRST` → `012345[REDACTED]`, `AUSTRALASIAWIDE…` → - // `AUSTRAL[REDACTED]`, a base32 OTP `ASIAMFRGGZDFMZTWQ2LK`. (The earlier "REGION_ASIA_PACIFIC_1 cannot + // processes: e.g. a 26-char ULID `012345AGPA…` → `012345[REDACTED]`, an all-caps `…ASIA…WIDE…` constant + // → `AUSTRAL[REDACTED]`, a base32 OTP `ASIA…`. (The earlier "…ASIA…-`_`-separated cannot // span it" reasoning was WRONG — it only considered `_`-separated forms and missed all-caps-contiguous // / ULID / base32.) Same word-fragment class we keep `\b` on for classic-github (`-ghs`), `xapp` (app), // `sk-`, `AIza` (AI), `ya29`. Effect: standalone / delimited / labeled AWS keys (`AKIA<16>`, diff --git a/test/unit/api/realtime/friday-event-payload-redactor.test.ts b/test/unit/api/realtime/friday-event-payload-redactor.test.ts index 61e81922a..7c2124558 100644 --- a/test/unit/api/realtime/friday-event-payload-redactor.test.ts +++ b/test/unit/api/realtime/friday-event-payload-redactor.test.ts @@ -233,8 +233,8 @@ describe("redactEventPayload — no over-redaction (NO DEGRADE)", () => { laugh: "laughs_per_minute_counter", note: "metric name walkthroughs_started_and_completed today", // AWS AKIA word-fragment: benign ULID / all-caps id with ASIA/AGPA glued after a word char. - ulid: "012345AGPABCDEFGHJKMNPQRST", - caps: "AUSTRALASIAWIDEDEPLOYMENT01", + ulid: "012345AGPABCDEFGHJKMNPQRST", // pragma: allowlist secret — benign ULID (AKIA-family scanner false positive) + caps: "AUSTRALASIAWIDEDEPLOYMENT01", // pragma: allowlist secret — benign all-caps constant (AKIA-family scanner false positive) }; const out = redactEventPayload(payload); expect(out).toEqual(payload); diff --git a/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts b/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts index 60c7c6aa1..fec7c7224 100644 --- a/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts +++ b/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts @@ -321,7 +321,7 @@ describe("FridayMemoryOutputFilter — SEC-SECRET-GLUED-PREFIX-001 glued distinc it("NO-DEGRADE: benign `…ghs_` identifiers survive content / metadata / tags byte-identical", () => { const benign = ["walkthroughs_completed_counter", "walkthroughs_completedThisWeek", "coughs_e3b0c44298fc1c14", "breakthroughs_this_quarter_list", "laughs_per_minute_counter", // AWS AKIA word-fragment: benign ULID / all-caps id with ASIA/AGPA glued after a word char. - "012345AGPABCDEFGHJKMNPQRST", "AUSTRALASIAWIDEDEPLOYMENT01"]; + "012345AGPABCDEFGHJKMNPQRST", "AUSTRALASIAWIDEDEPLOYMENT01"]; // pragma: allowlist secret — benign ULID/all-caps (AKIA-family scanner false positives) const out = filter.filterItem(makeItem({ content: "metric name walkthroughs_started_and_completed today", metadata: { walk: benign[0], breakt: benign[1], note: "keep" }, diff --git a/test/unit/security/friday-secret-shape-redactor.test.ts b/test/unit/security/friday-secret-shape-redactor.test.ts index fe041215a..2b4aca6f2 100644 --- a/test/unit/security/friday-secret-shape-redactor.test.ts +++ b/test/unit/security/friday-secret-shape-redactor.test.ts @@ -514,7 +514,7 @@ describe("friday-secret-shape-redactor", () => { "npm_config", // npm_ + short body with `_` "npm_config_cache is set", "550e8400-e29b-41d4-a716-446655440000", // a UUID - "abc123def456ghi789jkl012mno345pqr678stuv", // pure lowercase-alnum blob — no prefix reachable + "abc123def456ghi789jkl012mno345pqr678stuv", // pragma: allowlist secret — pure lowercase-alnum blob, no prefix reachable "aGVsbG8_d29ybGQ-dGhpc19pc19iZW5pZ24", // pragma: allowlist secret — base64url blob w/ `_` and `-`, not a secret shape "user_session_reference_identifier_v2", // snake_case id "AKIA is the aws access-key id prefix", // AKIA as a plain word (no 16-char body) @@ -522,10 +522,10 @@ describe("friday-secret-shape-redactor", () => { // after a word char — the AKIA branch keeps `\b`, so these are UNCHANGED (they were CORRUPTED while // AKIA was de-`\b`'d). (A DELIMITED `"ASIA<16>"` — quote boundary — is caught by BASE too, so it is // NOT a benign-unchanged case; only the GLUED forms are the fix's concern.) - "012345AGPABCDEFGHJKMNPQRST", // 26-char Crockford-base32 ULID (AGPA glued after digit `5`) - "AUSTRALASIAWIDEDEPLOYMENT01", // all-caps constant, ASIA glued after `L` - "EURASIAREGIONCODE0123456789", // all-caps, ASIA glued after `R` - "PROJECTAIDABUILDPIPELINE42X", // all-caps, AIDA glued after `T` + "012345AGPABCDEFGHJKMNPQRST", // pragma: allowlist secret — 26-char Crockford-base32 ULID (AGPA glued after digit `5`) + "AUSTRALASIAWIDEDEPLOYMENT01", // pragma: allowlist secret — all-caps constant, ASIA glued after `L` + "EURASIAREGIONCODE0123456789", // pragma: allowlist secret — all-caps, ASIA glued after `R` + "PROJECTAIDABUILDPIPELINE42X", // pragma: allowlist secret — all-caps, AIDA glued after `T` "9f8e7d6c5b4a3928170695f4e3d2c1b0", // pragma: allowlist secret — 32-hex id / git blob "/var/log/hf_service/npm_cache/output.log", // file path with hf_/npm_ short segments "GOCSPX_notasecret_underscore", // GOCSPX_ (underscore, not the required hyphen) @@ -584,9 +584,9 @@ describe("friday-secret-shape-redactor", () => { // glued directly after a word char (`xAKIA<16>`) is an ACCEPTED, documented gap. it("AWS AKIA: benign glued ULID/all-caps survives; DELIMITED/standalone key redacts; GLUED is an accepted gap", () => { // Benign all-caps / ULID with a fragment GLUED after a word char → UNCHANGED (the `\b` blocks it). - expect(redactSecretShapesInString("012345AGPABCDEFGHJKMNPQRST")).toBe("012345AGPABCDEFGHJKMNPQRST"); // ULID - expect(redactSecretShapesInString("AUSTRALASIAWIDEDEPLOYMENT01")).toBe("AUSTRALASIAWIDEDEPLOYMENT01"); - expect(redactSecretShapesInString("PROJECTAIDABUILDPIPELINE42X")).toBe("PROJECTAIDABUILDPIPELINE42X"); + expect(redactSecretShapesInString("012345AGPABCDEFGHJKMNPQRST")).toBe("012345AGPABCDEFGHJKMNPQRST"); // pragma: allowlist secret — benign ULID (AKIA-family scanner false positive) + expect(redactSecretShapesInString("AUSTRALASIAWIDEDEPLOYMENT01")).toBe("AUSTRALASIAWIDEDEPLOYMENT01"); // pragma: allowlist secret — benign all-caps constant + expect(redactSecretShapesInString("PROJECTAIDABUILDPIPELINE42X")).toBe("PROJECTAIDABUILDPIPELINE42X"); // pragma: allowlist secret — benign all-caps constant const KEY = seg("AKIA", "IOSFODNN7EXAMPLE"); // pragma: allowlist secret — AKIA + 16 [0-9A-Z] // DELIMITED / standalone / labeled AWS key IS still caught (whitespace, `=`, bare value). expect(redactSecretShapesInString(KEY)).toBe(M); diff --git a/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts b/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts index a4fa0890a..178ff25ef 100644 --- a/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts +++ b/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts @@ -232,8 +232,8 @@ describe("SEC-EVENT-REDACTION-001 — cross-sink parity (memory egress == audit cough: "coughs_e3b0c44298fc1c14", // ghs_ + content-hash suffix // AWS AKIA word-fragment: a benign ULID / all-caps id with ASIA/AGPA glued after a word char MUST // survive in BOTH sinks — the AKIA branch keeps `\b` (was corrupted while AKIA was de-`\b`'d). - ulid: "012345AGPABCDEFGHJKMNPQRST", - caps: "AUSTRALASIAWIDEDEPLOYMENT01", + ulid: "012345AGPABCDEFGHJKMNPQRST", // pragma: allowlist secret — benign ULID (AKIA-family scanner false positive) + caps: "AUSTRALASIAWIDEDEPLOYMENT01", // pragma: allowlist secret — benign all-caps constant (AKIA-family scanner false positive) }); it("glued distinctive-prefix credentials redact identically in the AUDIT sink and MEMORY redactDeep", async () => { @@ -253,8 +253,8 @@ describe("SEC-EVENT-REDACTION-001 — cross-sink parity (memory egress == audit expect(memoryValue.contig).toBe("walkthroughs_completedThisWeek"); expect(memoryValue.breakt).toBe("breakthroughs_this_quarter_list"); expect(memoryValue.cough).toBe("coughs_e3b0c44298fc1c14"); - expect(memoryValue.ulid).toBe("012345AGPABCDEFGHJKMNPQRST"); - expect(memoryValue.caps).toBe("AUSTRALASIAWIDEDEPLOYMENT01"); + expect(memoryValue.ulid).toBe("012345AGPABCDEFGHJKMNPQRST"); // pragma: allowlist secret — benign ULID (AKIA-family scanner false positive) + expect(memoryValue.caps).toBe("AUSTRALASIAWIDEDEPLOYMENT01"); // pragma: allowlist secret — benign all-caps constant (AKIA-family scanner false positive) // No credential body survives in either sink's serialization. for (const sink of [auditDetails, memoryValue]) { const json = JSON.stringify(sink); From 2c83630fa61bc363d23a8fa998d737fe3f10c7d5 Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.8 (1M context)" Date: Sat, 18 Jul 2026 16:43:14 -0700 Subject: [PATCH 6/7] fix(security): split GitHub/AWS secret-detector alternations by prefix; de-`\b` only unambiguous members (ghp/ghr/AKIA) to catch glued credentials (SEC-SECRET-GLUED-PREFIX-001 P1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Advisor P1 audit found that keeping the leading `\b` on the WHOLE GitHub-classic and AWS alternations left real credentials glued after a word char UNMATCHED (`keyghp_<36>`, `keyAKIA<16>`) — they survived the detector, realtime redactor, memory filter, and persisted audit JSONL. "Accepted gap" does not satisfy SEC-EVENT-REDACTION-001. Fix: SPLIT each alternation into two patterns and drop the leading `\b` ONLY on the members that are provably NOT a benign word / ULID / acronym fragment: - GitHub classic {ghp,ghr} → de-`\b` (0 dict word-endings). {gho,ghs,ghu,github_pat_} keep `\b` (word-endings sor-GHO / walkthrou-GHS / Ra-GHU; "github" is a word) — the mixed-case body lets a lowercase snake_case identifier collide, so even obscure word-endings matter. - AWS {AKIA} → de-`\b` (has `I` → not ULID-constructible; only obscure lowercase word-endings that never form all-caps 20+ runs; no common all-caps acronym). {ASIA,AGPA,AIDA,AROA,AIPA, ANPA,ANVA} keep `\b` (Eur-ASIA word / AGPA·ANPA·ANVA ULID-constructible / AIDA·AIPA→AIPAC acronym / AOTE-AROA place) — the uppercase-only body only collides inside all-caps runs. Glued `keyghp_`/`keyghr_`/`keyAKIA` are now redacted at the credential subspan (benign lead preserved) at the detector, event-payload redactor, memory output filter, and audit-JSONL sink; the benign no-degrade corpus (walkthroughs/coughs, ULIDs, ASIA/AIDA/AGPA all-caps, AIPAC/AOTEAROA /CANVAS, sorgho_/raghu_) is byte-identical vs base. Bodies stay base62/`[0-9A-Z]{16}` and the trailing `\b` is unchanged (open-ended `{16,}`, never tightened, so no real token slips it). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01JhCDTLngxAf6MDh4Bfcd48 --- src/security/friday-secret-shape-redactor.ts | 96 +++++++++----- .../friday-event-payload-redactor.test.ts | 8 +- ...friday-memory-output-filter-secret.test.ts | 24 ++++ .../friday-secret-shape-redactor.test.ts | 119 ++++++++++++------ ...nt-redaction-001-cross-sink-parity.test.ts | 12 +- 5 files changed, 188 insertions(+), 71 deletions(-) diff --git a/src/security/friday-secret-shape-redactor.ts b/src/security/friday-secret-shape-redactor.ts index 9641ef1b9..ae5bdf3c0 100644 --- a/src/security/friday-secret-shape-redactor.ts +++ b/src/security/friday-secret-shape-redactor.ts @@ -192,22 +192,42 @@ const SECRET_CONTENT_PATTERNS: readonly SecretContentPattern[] = [ }, // GitHub tokens: classic `gh?_` prefixes AND fine-grained `github_pat_`. // - // GLUED-PREFIX (SEC-SECRET-GLUED-PREFIX-001): the GitHub branch KEEPS its leading `\b` — UNLIKE the 10 - // other de-`\b`'d shapes, the classic prefixes are common ENGLISH WORD-FRAGMENTS: `ghs` ends - // walkthrou-GHS, breakthrou-GHS, cou-GHS, hi-GHS, lau-GHS, rou-GHS, wei-GHS, plou-GHS, borou-GHS; and - // `github_pat_` contains the word "github". So dropping the boundary would over-redact benign - // `_<16+ CONTIGUOUS base62 run>` content this SHARED egress detector processes — - // even with a base62 body: `walkthroughs_completedThisWeek` → `ghs_completedThisWeek` (17 base62) or - // `coughs_e3b0c44298fc1c14` (a content-hash suffix) would be corrupted. Because the collision surface - // is a word FRAGMENT (not a distinctive token prefix), `\b` is REQUIRED. Effect: a DELIMITED / - // standalone / labeled classic token (`ghp_`, `token: ghp_…`, `apikey=ghp_…`) is STILL caught - // (the common case); only a classic token glued DIRECTLY after a word char (`keyghp_…`) is an - // ACCEPTED, documented gap — exactly like the kept-`\b` `sk-`/`AIza`/`ya29.`/`eyJ`/`xapp-` prefixes. - // The classic body is still BASE62 (`[A-Za-z0-9]`, real-token shape) so a standalone `ghs_a_b_c` - // snake_case identifier breaks at the first `_`; the `github_pat_` branch keeps its `_`-inclusive body - // (a fine-grained PAT's body legitimately has `_`). + // GLUED-PREFIX (SEC-SECRET-GLUED-PREFIX-001): the classic set {gho,ghp,ghr,ghs,ghu} is SPLIT by prefix, + // and the leading `\b` is DROPPED ONLY on the members that are PROVABLY not a benign word / identifier + // fragment — so a real token glued DIRECTLY after a word char (`keyghp_…`, the Advisor P1 case) is + // caught, WITHOUT re-introducing the demonstrated benign over-redaction. The classic body is MIXED-CASE + // base62 with a `_` separator, so the collision surface is a LOWERCASE snake_case identifier ending in + // the member (`_<16+ contiguous base62>`) — even an obscure lowercase word matters here (UNLIKE + // the uppercase-only AWS body below). Classification (grep /usr/share/dict/words for word-endings): + // • ghp — 0 dict word-endings, not a benign fragment → UNAMBIGUOUS → `\b` DROPPED (a glued + // `keyghp_<36>` classic PAT is now caught; this is the Advisor P1 canary). + // • ghr — 0 dict word-endings, not a benign fragment → UNAMBIGUOUS → `\b` DROPPED (a glued + // `keyghr_<…>` refresh token is now caught). + // • ghs — a PLURAL word-ending (walkthrou-GHS, cou-GHS, breakthrou-GHS, hi-GHS, lau-GHS, si-GHS, + // trou-GHS, …) → AMBIGUOUS → `\b` KEPT (else `walkthroughs_completedThisWeek` — a CONTIGUOUS 17 + // base62 run — over-redacts to `walkthrou[REDACTED]`). + // • gho — dict word-endings (sor-GHO, a millet; San-GHO, a language) → a lowercase + // `sorgho_` would over-redact → AMBIGUOUS → `\b` KEPT. + // • ghu — dict word-ending (Ra-GHU, a common given name) → `raghu_` would over-redact → + // AMBIGUOUS → `\b` KEPT. + // • github_pat_ — contains the natural word "github" → AMBIGUOUS → `\b` KEPT. + // Effect for the KEPT members {gho,ghs,ghu,github_pat_}: a DELIMITED / standalone / labeled classic + // token (`gho_`, `token: ghu_…`, `apikey=ghs_…`) is STILL caught by the assignment/whitespace + // paths (the common case); only a token glued DIRECTLY after a word char stays an ACCEPTED gap for + // THOSE word-fragment prefixes (like `sk-`/`AIza`/`ya29.`/`eyJ`). For the DROPPED members {ghp,ghr} a + // glued token is now REDACTED at exactly the credential subspan (the benign leading char survives + // byte-for-byte). Bodies stay BASE62 (`[A-Za-z0-9]`) so a snake_case identifier breaks at the first `_` + // (open-ended `{16,}` — never tightened to an exact length, so a longer real token cannot slip the + // trailing `\b`); the `github_pat_` branch keeps its `_`-inclusive body (a fine-grained PAT legitimately + // has `_` in its body). { - pattern: /\b(?:gh[opsru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{16,})\b/gu, + // UNAMBIGUOUS classic prefixes — leading `\b` DROPPED so a glued `ghp_<36>` / `ghr_<…>` is caught. + pattern: /gh[pr]_[A-Za-z0-9]{16,}\b/gu, + sensitiveSpan: wholeMatchSpan, + }, + { + // AMBIGUOUS classic prefixes (word-fragment endings gho/ghs/ghu) + fine-grained `github_pat_` — leading `\b` KEPT. + pattern: /\b(?:gh[osu]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{16,})\b/gu, sensitiveSpan: wholeMatchSpan, }, // Provider hyphen-prefixed API keys: OpenAI `sk-` / `sk-proj-`, `rk-`, `ak-`, xAI `xai-`. The @@ -241,20 +261,40 @@ const SECRET_CONTENT_PATTERNS: readonly SecretContentPattern[] = [ // AWS access-key id (`AKIA…` long-term + STS/temporary variants). NB: the AWS SECRET access key is // a shapeless 40-char base64 with no prefix — it is caught by its KEY NAME (`secretAccessKey`, // in the sensitive-field-name set), NOT a shape, so it is not (and must not be) a content pattern. - // GLUED-PREFIX: leading `\b` KEPT deliberately (UNLIKE the base62 vendor prefixes). The `A(?:KIA|SIA|…)` - // alternation embeds ENGLISH-WORD / base32-constructible fragments — `ASIA` (Asia / Eurasia / - // Australasia), `AIDA`, and `AGPA`/`ANPA`/`ANVA` (constructible from Crockford base32) — AND its body is - // `[0-9A-Z]{16}`, i.e. UPPERCASE + DIGIT, exactly the alphabet of an all-caps constant or a ULID. So - // de-`\b` over-redacts benign all-caps / ULID / base32 identifiers this SHARED egress detector - // processes: e.g. a 26-char ULID `012345AGPA…` → `012345[REDACTED]`, an all-caps `…ASIA…WIDE…` constant - // → `AUSTRAL[REDACTED]`, a base32 OTP `ASIA…`. (The earlier "…ASIA…-`_`-separated cannot - // span it" reasoning was WRONG — it only considered `_`-separated forms and missed all-caps-contiguous - // / ULID / base32.) Same word-fragment class we keep `\b` on for classic-github (`-ghs`), `xapp` (app), - // `sk-`, `AIza` (AI), `ya29`. Effect: standalone / delimited / labeled AWS keys (`AKIA<16>`, - // `token AKIA…`, `apikey=AKIA…` — AWS keys are ~always delimited in env/config) are STILL caught; only - // a key glued DIRECTLY after a word char (`xAKIA<16>`) is an ACCEPTED, documented gap. + // + // GLUED-PREFIX (SEC-SECRET-GLUED-PREFIX-001): the AWS family is SPLIT by prefix and the leading `\b` is + // DROPPED ONLY on `AKIA`. The AWS body is `[0-9A-Z]{16}` — UPPERCASE + DIGIT, i.e. the alphabet of an + // all-caps constant / ULID / base32 blob — so (UNLIKE the mixed-case github body above) a member + // over-redacts ONLY when it sits inside an ALL-CAPS 20+ contiguous run; a lowercase prose word never + // collides. A member is de-`\b`'d ONLY if it fails ALL of: (a) a COMMON word-ending that forms all-caps + // runs, (b) ULID-constructible (Crockford base32 = 0-9 A-Z minus I,L,O,U — a member of only Crockford + // chars can sit mid-ULID), and (c) a common all-caps ACRONYM / fragment: + // • AKIA — has `I` → NOT ULID-constructible; only OBSCURE lowercase word-endings (aph-AKIA, + // leucopl-AKIA, Lat-AKIA — medical / geographic, never written as ALL-CAPS 20+ runs); no common + // all-caps acronym source → fails (a),(b),(c) → UNAMBIGUOUS → `\b` DROPPED (a glued `keyAKIA<16>` + // is now caught; this is the Advisor P1 canary). + // • ASIA — has `I` (not ULID) BUT a COMMON word-ending (Eur-ASIA, Austral-ASIA, aph-ASIA) written + // all-caps in region constants (…ASIA…, AUSTRALASIA…, EURASIA…) → fails (a),(c) → AMBIGUOUS → `\b` KEPT. + // • AGPA / ANPA / ANVA — Crockford-only (no I/L/O/U) → ULID-CONSTRUCTIBLE (sit mid-ULID, e.g. + // `012345AGPA…`); ANVA is also inside C-ANVA-S → fails (b) → AMBIGUOUS → `\b` KEPT. + // • AIDA — has `I` (not ULID) BUT a common all-caps ACRONYM (the AIDA marketing model, PROJECT-AIDA, + // the opera) → fails (c) → AMBIGUOUS → `\b` KEPT. + // • AROA — has `O` (not ULID) BUT the place name AOTE-AROA (all-caps-able in NZ constants) → fails + // (a),(c) → AMBIGUOUS → `\b` KEPT. + // • AIPA — has `I` (not ULID) BUT a prefix of the common all-caps acronym / org AIPA-C + // (`AIPACPOLICYCONFERENCE…`) → fails (c) → AMBIGUOUS → `\b` KEPT. + // Effect: a DELIMITED / standalone / labeled AWS key of ANY prefix is STILL caught (the common case — + // AWS keys are ~always delimited in env/config); a glued `keyAKIA<16>` is now REDACTED at exactly the + // credential subspan, while a glued benign all-caps / ULID carrying an AMBIGUOUS fragment + // (`012345AGPA…`, `…ASIA…WIDE…`, `PROJECTAIDA…`) is UNCHANGED (`AKIA` never appears in those blobs). + { + // UNAMBIGUOUS — leading `\b` DROPPED so a glued `AKIA<16>` is caught. + pattern: /AKIA[0-9A-Z]{16}\b/gu, + sensitiveSpan: wholeMatchSpan, + }, { - pattern: /\bA(?:KIA|SIA|GPA|IDA|ROA|IPA|NPA|NVA)[0-9A-Z]{16}\b/gu, + // AMBIGUOUS (word-ending / ULID-constructible / acronym fragments) — leading `\b` KEPT. + pattern: /\bA(?:SIA|GPA|IDA|ROA|IPA|NPA|NVA)[0-9A-Z]{16}\b/gu, sensitiveSpan: wholeMatchSpan, }, // Google API key — `AIza` + 35 chars of `[0-9A-Za-z_-]` (documented 39-char format). diff --git a/test/unit/api/realtime/friday-event-payload-redactor.test.ts b/test/unit/api/realtime/friday-event-payload-redactor.test.ts index 7c2124558..76c8bd993 100644 --- a/test/unit/api/realtime/friday-event-payload-redactor.test.ts +++ b/test/unit/api/realtime/friday-event-payload-redactor.test.ts @@ -166,11 +166,15 @@ describe("redactEventPayload — secret shapes (no regression)", () => { const HF = seg("hf_", "AbCdEfGhIjKlMnOpQrStUvWxYz01234567"); // pragma: allowlist secret — 34 base62 const GSK = seg("gsk_", "abcdefghijklmnopqrstuvwxyz0123456789ABCDwx"); // pragma: allowlist secret — 42 base62 const GLPAT = seg("glpat-", "ABCdef0123456789ghijkLMNop"); // pragma: allowlist secret + // SEC-SECRET-GLUED-PREFIX-001 P1: GitHub classic `ghp_` and AWS `AKIA` were SPLIT out and de-`\b`'d — + // a real credential glued after a word char (`keyghp_<36>`, `keyAKIA<16>`) must now be caught on-wire. + const GHP = seg("ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"); // pragma: allowlist secret — 36 base62 + const AKIA = seg("AKIA", "IOSFODNN7EXAMPLE"); // pragma: allowlist secret — AKIA + 16 [0-9A-Z] const out = redactEventPayload({ - log: seg("deploy key", HF, " and x", GSK, " and id", GLPAT, " done"), + log: seg("deploy key", HF, " and x", GSK, " and id", GLPAT, " and gh", GHP, " and aws", AKIA, " done"), }); const s = serialize(out); - for (const cred of [HF, GSK, GLPAT]) expect(s, cred).not.toContain(cred); + for (const cred of [HF, GSK, GLPAT, GHP, AKIA]) expect(s, cred).not.toContain(cred); expect(s).toContain("[REDACTED]"); // Benign surrounding + glued leading chars survive (only the credential subspan is masked). expect(s).toContain("deploy key"); diff --git a/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts b/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts index fec7c7224..9c7ae6124 100644 --- a/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts +++ b/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts @@ -315,6 +315,30 @@ describe("FridayMemoryOutputFilter — SEC-SECRET-GLUED-PREFIX-001 glued distinc expect(out.content).toBe(seg("ref key", "hf_docs")); }); + // SEC-SECRET-GLUED-PREFIX-001 P1: GitHub classic `ghp_`/`ghr_` and AWS `AKIA` were SPLIT out of their + // alternations and de-`\b`'d (they are provably NOT benign word/ULID/acronym fragments), so a real + // credential GLUED after a word char is now caught in every output-filter leg. RED on 7021926c (glued + // token survives content + metadata + kept as a tag); GREEN after. + it("redacts GLUED ghp_/ghr_/AKIA credentials in content / metadata / tags (P1 split), benign lead preserved", () => { + const GHP = seg("ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"); // pragma: allowlist secret — 36 base62 + const AKIA = seg("AKIA", "IOSFODNN7EXAMPLE"); // pragma: allowlist secret — AKIA + 16 [0-9A-Z] + for (const [glueWord, body] of [["key", GHP], ["aws", AKIA]] as const) { + const glued = seg(glueWord, body); + // content leg — only the credential subspan is masked, the glued benign word survives. + const c = filter.filterItem(makeItem({ content: seg("token ", glued, " now") })); + expect(c.content).toBe(`token ${glueWord}${M} now`); + expect(c.content).not.toContain(body); + // metadata value leg under a non-sensitive key. + const m = filter.filterItem(makeItem({ metadata: { ref: glued, note: "keep" } })); + expect((m.metadata as { ref: string; note: string }).ref).toBe(`${glueWord}${M}`); + expect((m.metadata as { note: string }).note).toBe("keep"); + // tag leg — the glued-credential tag is dropped, benign tags preserved. + const t = filter.filterItem(makeItem({ tags: ["ok", glued, "fine"] })); + expect(JSON.stringify(t.tags)).not.toContain(body); + expect(t.tags).toEqual(["ok", "fine"]); + } + }); + // NO-DEGRADE (round-2): benign snake_case words ending in `ghs`/etc before `_` MUST survive through // EVERY output-filter leg — the github-classic base62 body breaks at the `_`. The first-round // `[A-Za-z0-9_]` body corrupted these (`walkthroughs_completed_counter` → `walkthrou[REDACTED_SECRET]`). diff --git a/test/unit/security/friday-secret-shape-redactor.test.ts b/test/unit/security/friday-secret-shape-redactor.test.ts index 2b4aca6f2..ca3bdefd8 100644 --- a/test/unit/security/friday-secret-shape-redactor.test.ts +++ b/test/unit/security/friday-secret-shape-redactor.test.ts @@ -434,10 +434,16 @@ describe("friday-secret-shape-redactor", () => { ["SendGrid SG.<22>.<43>", SG_SECRET], // pragma: allowlist secret ["Square sq0atp-", seg("sq0atp-", "0123456789abcdefghijklABCDwxyz")], // pragma: allowlist secret ["Square sq0csp-", seg("sq0csp-", "0123456789abcdefghijklABCDwxyz")], // pragma: allowlist secret - // NB: GitHub classic `gh[opsru]_` / `github_pat_` and AWS `AKIA…` are NOT here — their prefixes are - // English-word / base32 fragments (`-ghs`, "github", `ASIA`/`AIDA`), so they KEEP `\b` (glued case is - // an accepted gap); see KEPT. The 9 shapes above are non-word-fragment vendor prefixes with a body - // that excludes `_`/`-` (or an ultra-distinctive prefix), so they safely stay de-`\b`'d. + // GitHub classic {ghp,ghr} and AWS {AKIA} were SPLIT out of their alternations and de-`\b`'d + // (SEC-SECRET-GLUED-PREFIX-001 P1 fix): `ghp`/`ghr` have ZERO dict word-endings; `AKIA` has `I` + // (not ULID-constructible), only obscure lowercase word-endings (aphakia/leucoplakia — never + // all-caps 20+ runs), and no common all-caps acronym → all three are provably NOT benign fragments, + // so a glued `keyghp_<36>` / `keyghr_<…>` / `keyAKIA<16>` real credential is now CAUGHT. The + // AMBIGUOUS members {gho,ghs,ghu,github_pat_} and {ASIA,AGPA,AIDA,AROA,AIPA,ANPA,ANVA} KEEP `\b` + // (word-ending / ULID / acronym fragments) — see KEPT. + ["GitHub classic ghp_ (0 word-endings)", seg("ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")], // pragma: allowlist secret — 36 base62 + ["GitHub classic ghr_ (0 word-endings)", seg("ghr_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")], // pragma: allowlist secret — 36 base62 + ["AWS AKIA (not ULID/word/acronym)", seg("AKIA", "IOSFODNN7EXAMPLE")], // pragma: allowlist secret — AKIA + 16 [0-9A-Z] ["Slack xoxb-", seg("xoxb-", "EXAMPLENOTAREALSLACKTOKEN")], // pragma: allowlist secret ]; @@ -484,11 +490,13 @@ describe("friday-secret-shape-redactor", () => { ["xai- (…xai)", seg("xai-", "abcdefghijklmnop0123456789"), seg("proxai-", "abcdefghijklmnop0123456789")], // pragma: allowlist secret ["Stripe sk_live_ (desk_/risk_)", seg("sk", "_live_", "0123456789abcdefABCD"), seg("desk", "_live_", "0123456789abcdefABCD")], // pragma: allowlist secret ["github_pat_ (natural phrase)", seg("github_pat_", "11ABCDE0aBcDeFgHiJkL0"), seg("my", "github_pat_", "reference0token0id00")], // pragma: allowlist secret - // GitHub classic — `-ghs`/`-gho`/… are word-endings (walkthroughs), so a glued form is NOT caught. - ["GitHub classic ghp_ (…ghs word-ending)", seg("ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"), seg("walkthroughs_", "completedThisWeekX")], // pragma: allowlist secret - // AWS AKIA — `ASIA`/`AIDA` are English words + base32-constructible, body is all-caps/digit (ULID - // alphabet), so a glued form (a 26-char ULID `012345AGPA…`) is NOT caught; standalone still is. - ["AWS AKIA (ASIA/AIDA word-fragment)", seg("AKIA", "IOSFODNN7EXAMPLE"), "012345AGPABCDEFGHJKMNPQRST"], // pragma: allowlist secret — glued-benign is a ULID + // GitHub classic AMBIGUOUS {gho,ghs,ghu} — word-endings (sor-GHO / walkthrou-GHS / Ra-GHU), so a + // glued form is NOT caught; standalone still is. (ghp/ghr were de-`\b`'d — see DEBOUNDED.) + ["GitHub classic gho_ (…gho/ghs word-ending)", seg("gho_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"), seg("walkthroughs_", "completedThisWeekX")], // pragma: allowlist secret + // AWS AMBIGUOUS {ASIA,AGPA,…} — ASIA is an English word + AGPA is ULID-constructible; the body is + // all-caps/digit (ULID alphabet), so a glued form (a 26-char ULID `012345AGPA…`) is NOT caught; + // standalone still is. (AKIA was de-`\b`'d — see DEBOUNDED.) + ["AWS AMBIGUOUS ASIA/AGPA (word/ULID fragment)", seg("ASIA", "JEXAMPLE01234XYZ"), "012345AGPABCDEFGHJKMNPQRST"], // pragma: allowlist secret — glued-benign is a ULID ["Slack xapp- (maxapp-)", seg("xapp-", "1-A0123ABCD-4567890123"), "maxapp-config-value-here"], // pragma: allowlist secret ["Google AIza (openAIza…)", seg("AIza", "SyDabcdefghijklmnopqrstuvwxyz012345"), seg("open", "AIza", "SyDabcdefghijklmnopqrstuvwxyz012345")], // pragma: allowlist secret ["Google ya29. (maya29.field)", seg("ya29.", "a0AfBbyDtestTokenValue0123456789ABCDEF"), "maya29.profile_image_url_field_v2"], // pragma: allowlist secret @@ -526,6 +534,11 @@ describe("friday-secret-shape-redactor", () => { "AUSTRALASIAWIDEDEPLOYMENT01", // pragma: allowlist secret — all-caps constant, ASIA glued after `L` "EURASIAREGIONCODE0123456789", // pragma: allowlist secret — all-caps, ASIA glued after `R` "PROJECTAIDABUILDPIPELINE42X", // pragma: allowlist secret — all-caps, AIDA glued after `T` + // Negative controls for the KEPT-`\b` AWS members whose fragment is NOT ULID nor a common word but + // IS inside a common all-caps acronym / place name (criterion (c)) — these MUST survive: + "AIPACPOLICYCONFERENCE2024ABC", // pragma: allowlist secret — AIPA is a prefix of the all-caps org AIPAC + "AOTEAROANEWZEALANDGOVT01ABCD", // pragma: allowlist secret — AROA is inside the place name AOTEAROA + "OPENCANVASRENDERINGCONTEXT2D", // pragma: allowlist secret — ANVA is inside CANVAS (also ULID-constructible) "9f8e7d6c5b4a3928170695f4e3d2c1b0", // pragma: allowlist secret — 32-hex id / git blob "/var/log/hf_service/npm_cache/output.log", // file path with hf_/npm_ short segments "GOCSPX_notasecret_underscore", // GOCSPX_ (underscore, not the required hyphen) @@ -545,6 +558,10 @@ describe("friday-secret-shape-redactor", () => { "troughs_index", "sighs_and_weighs_and_doughs", // multiple ghs-ending words "metric name walkthroughs_started_and_completed today", // free-text sentence + // Negative controls for the KEPT-`\b` github members gho/ghu (word-endings sor-GHO / Ra-GHU): a + // lowercase `_<16+ contiguous base62>` MUST survive (would over-redact if these were de-`\b`'d). + "sorgho_yieldPerHectare2024xx", // gho_ + 21 contiguous base62 — only the leading `\b` closes this + "raghu_authTokenReferenceValueV2", // ghu_ + contiguous base62 run ]; it("NO-DEGRADE: the benign-identifier corpus is returned byte-identical (zero over-redaction)", () => { @@ -554,47 +571,71 @@ describe("friday-secret-shape-redactor", () => { } }); - // GitHub-classic sensitivity boundary (round-3): the classic prefixes are common English - // WORD-FRAGMENTS (`-ghs` ends walkthroughs/coughs/highs; "github" is a word), so — unlike the 10 - // other de-`\b`'d shapes — the classic branch KEEPS `\b`. A benign `<-ghs word>_` is UNCHANGED (the boundary blocks it, even for a contiguous run that a base62 body alone did - // not close). A DELIMITED / standalone / labeled classic token IS still caught (the common case); a - // classic token glued DIRECTLY after a word char is an ACCEPTED, documented gap (like sk-/AIza/ya29.). - it("GitHub-classic: benign `…ghs_` survives; DELIMITED/standalone token redacts; GLUED is an accepted gap", () => { - // Benign — the leading `\b` blocks the match; UNCHANGED even for a CONTIGUOUS 17+ base62 run. + // GitHub-classic sensitivity boundary (SEC-SECRET-GLUED-PREFIX-001 P1 split): the classic alternation + // is SPLIT by prefix. {ghp,ghr} have ZERO dict word-endings → de-`\b`'d (a glued token is now CAUGHT); + // {gho,ghs,ghu} are word-endings (sor-GHO / walkthrou-GHS / Ra-GHU) and `github_pat_` contains + // "github" → KEEP `\b` (a glued token stays an ACCEPTED gap). Benign `<-ghs word>_` is UNCHANGED for the KEPT members even for a contiguous run a base62 body alone did not close; + // a DELIMITED / standalone / labeled classic token IS still caught for EVERY prefix. + it("GitHub-classic split: ghp/ghr GLUED redacts (P1); benign `…ghs_` survives; gho/ghs/ghu GLUED is an accepted gap", () => { + // Benign — the KEPT `\b` blocks the match; UNCHANGED even for a CONTIGUOUS 17+ base62 run. expect(redactSecretShapesInString("walkthroughs_completed_counter")).toBe("walkthroughs_completed_counter"); expect(redactSecretShapesInString("walkthroughs_completedThisWeek")).toBe("walkthroughs_completedThisWeek"); // 17 contiguous base62 expect(redactSecretShapesInString("coughs_e3b0c44298fc1c14")).toBe("coughs_e3b0c44298fc1c14"); // hash suffix expect(redactSecretShapesInString("highs_thresholdValueConfig1")).toBe("highs_thresholdValueConfig1"); - const TOK = seg("ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"); // pragma: allowlist secret — 36 base62 - // DELIMITED / standalone / labeled classic token IS still caught (whitespace, `=`, bare value). - expect(redactSecretShapesInString(TOK)).toBe(M); - expect(redactSecretShapesInString(`token ${TOK} used`)).toBe(`token ${M} used`); - expect(redactSecretShapesInString(`apikey=${TOK}`)).toBe(`apikey=${M}`); - // GLUED directly after a word char is an ACCEPTED gap for these word-fragment prefixes. - expect(redactSecretShapesInString(seg("key", TOK))).toBe(seg("key", TOK)); - expect(findSecretShapeSpans(seg("key", TOK))).toEqual([]); - }); - - // AWS AKIA sensitivity boundary (round-4): the alternation embeds English-word / base32-constructible - // fragments (`ASIA`/`AIDA`/`AGPA`) and its body is `[0-9A-Z]{16}` (the ULID / all-caps-constant - // alphabet), so — like classic-github — the AKIA branch KEEPS `\b`. A benign all-caps / ULID / base32 - // id where the fragment is GLUED after a word char is UNCHANGED. A DELIMITED / standalone / labeled - // AWS key IS still caught (the common case — AWS keys are ~always delimited in env/config); a key - // glued directly after a word char (`xAKIA<16>`) is an ACCEPTED, documented gap. - it("AWS AKIA: benign glued ULID/all-caps survives; DELIMITED/standalone key redacts; GLUED is an accepted gap", () => { - // Benign all-caps / ULID with a fragment GLUED after a word char → UNCHANGED (the `\b` blocks it). + const B62 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; // 36 base62 + // DE-`\b`'d {ghp,ghr}: DELIMITED still caught AND a GLUED token is now REDACTED (P1 canary), the + // benign leading char preserved byte-for-byte; the span is exactly the credential. + for (const p of ["ghp_", "ghr_"]) { + const TOK = seg(p, B62); // pragma: allowlist secret + expect(redactSecretShapesInString(TOK), `${p} standalone`).toBe(M); + expect(redactSecretShapesInString(`token ${TOK} used`), `${p} delimited`).toBe(`token ${M} used`); + expect(redactSecretShapesInString(seg("key", TOK)), `${p} GLUED`).toBe(`key${M}`); // P1: now caught + const spans = findSecretShapeSpans(seg("key", TOK)); + expect(spans.length, `${p} glued span`).toBe(1); + expect(seg("key", TOK).slice(spans[0]!.start, spans[0]!.end), `${p} glued span body`).toBe(TOK); + } + // KEPT-`\b` {gho,ghs,ghu}: DELIMITED / standalone still caught, but a GLUED token stays an ACCEPTED + // gap (these prefixes are benign word-endings, so de-`\b` would over-redact `sorgho_…`/`…ghs_…`). + for (const p of ["gho_", "ghs_", "ghu_"]) { + const TOK = seg(p, B62); // pragma: allowlist secret + expect(redactSecretShapesInString(TOK), `${p} standalone`).toBe(M); + expect(redactSecretShapesInString(`apikey=${TOK}`), `${p} labeled`).toBe(`apikey=${M}`); + expect(redactSecretShapesInString(seg("key", TOK)), `${p} GLUED gap`).toBe(seg("key", TOK)); + expect(findSecretShapeSpans(seg("key", TOK)), `${p} glued span`).toEqual([]); + } + }); + + // AWS sensitivity boundary (SEC-SECRET-GLUED-PREFIX-001 P1 split): the AWS alternation is SPLIT by + // prefix. `AKIA` (has `I` → not ULID; only obscure lowercase word-endings; no common all-caps acronym) + // is de-`\b`'d → a glued `keyAKIA<16>` is now CAUGHT (P1 canary). The AMBIGUOUS members + // {ASIA,AGPA,AIDA,AROA,AIPA,ANPA,ANVA} — word-endings (Eur-ASIA), ULID-constructible (AGPA/ANPA/ANVA), + // or acronym fragments (AIDA, AIPA→AIPAC, AROA→AOTEAROA) — KEEP `\b`, so a benign all-caps / ULID id + // where the fragment is GLUED after a word char is UNCHANGED. A DELIMITED / standalone / labeled AWS + // key IS still caught for EVERY prefix (the common case — AWS keys are ~always delimited in env/config). + it("AWS split: AKIA GLUED redacts (P1); benign glued ULID/all-caps survives; ASIA/AGPA/AIDA/… GLUED is an accepted gap", () => { + const B16 = "JEXAMPLE01234XYZ"; // 16 [0-9A-Z] + // Benign all-caps / ULID with an AMBIGUOUS fragment GLUED after a word char → UNCHANGED (the `\b` blocks it). expect(redactSecretShapesInString("012345AGPABCDEFGHJKMNPQRST")).toBe("012345AGPABCDEFGHJKMNPQRST"); // pragma: allowlist secret — benign ULID (AKIA-family scanner false positive) expect(redactSecretShapesInString("AUSTRALASIAWIDEDEPLOYMENT01")).toBe("AUSTRALASIAWIDEDEPLOYMENT01"); // pragma: allowlist secret — benign all-caps constant expect(redactSecretShapesInString("PROJECTAIDABUILDPIPELINE42X")).toBe("PROJECTAIDABUILDPIPELINE42X"); // pragma: allowlist secret — benign all-caps constant + // DE-`\b`'d AKIA: DELIMITED still caught AND a GLUED key is now REDACTED (P1 canary); leading char preserved. const KEY = seg("AKIA", "IOSFODNN7EXAMPLE"); // pragma: allowlist secret — AKIA + 16 [0-9A-Z] - // DELIMITED / standalone / labeled AWS key IS still caught (whitespace, `=`, bare value). expect(redactSecretShapesInString(KEY)).toBe(M); expect(redactSecretShapesInString(`token ${KEY} used`)).toBe(`token ${M} used`); expect(redactSecretShapesInString(`apikey=${KEY}`)).toBe(`apikey=${M}`); - // GLUED directly after a word char is an ACCEPTED gap for this word-fragment prefix. - expect(redactSecretShapesInString(seg("x", KEY))).toBe(seg("x", KEY)); - expect(findSecretShapeSpans(seg("x", KEY))).toEqual([]); + expect(redactSecretShapesInString(seg("key", KEY))).toBe(`key${M}`); // P1: glued now caught + const spans = findSecretShapeSpans(seg("key", KEY)); + expect(spans.length).toBe(1); + expect(seg("key", KEY).slice(spans[0]!.start, spans[0]!.end)).toBe(KEY); + // KEPT-`\b` AMBIGUOUS members: DELIMITED / standalone still caught, but a GLUED key stays an ACCEPTED gap. + for (const p of ["ASIA", "AGPA", "AIDA", "AROA", "AIPA", "ANPA", "ANVA"]) { + const K = seg(p, B16); // pragma: allowlist secret + expect(redactSecretShapesInString(K), `${p} standalone`).toBe(M); + expect(redactSecretShapesInString(`apikey=${K}`), `${p} labeled`).toBe(`apikey=${M}`); + expect(redactSecretShapesInString(seg("x", K)), `${p} GLUED gap`).toBe(seg("x", K)); + expect(findSecretShapeSpans(seg("x", K)), `${p} glued span`).toEqual([]); + } }); }); }); diff --git a/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts b/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts index 178ff25ef..1b2ee7903 100644 --- a/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts +++ b/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts @@ -218,11 +218,17 @@ describe("SEC-EVENT-REDACTION-001 — cross-sink parity (memory egress == audit const GSK_GLUED = seg("x", "gsk_", "abcdefghijklmnopqrstuvwxyz0123456789ABCDwx"); // pragma: allowlist secret const GLPAT_GLUED = seg("id", "glpat-", "ABCdef0123456789ghijkLMNop"); // pragma: allowlist secret const HF_GLUED = seg("key", "hf_", HF_BODY); // pragma: allowlist secret + // SEC-SECRET-GLUED-PREFIX-001 P1: GitHub classic `ghp_` and AWS `AKIA` were SPLIT out and de-`\b`'d, so + // a glued real credential must be caught identically in the AUDIT sink (persisted JSONL) and memory egress. + const GHP_GLUED = seg("key", "ghp_", "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"); // pragma: allowlist secret — 36 base62 + const AKIA_GLUED = seg("aws", "AKIA", "IOSFODNN7EXAMPLE"); // pragma: allowlist secret — AKIA + 16 [0-9A-Z] const GLUED_PAYLOAD = (): Record => ({ // non-sensitive KEY NAMES so the key-name nuke does NOT fire — the glued SHAPE must be caught. hfref: HF_GLUED, gskref: GSK_GLUED, glref: GLPAT_GLUED, + ghpref: GHP_GLUED, + akiaref: AKIA_GLUED, note: "just a note", // NO-DEGRADE: benign words ending in `ghs`/etc before `_` MUST survive in BOTH sinks — the // github-classic branch keeps `\b`, so a `<-ghs word>_` is not over-matched. @@ -242,10 +248,12 @@ describe("SEC-EVENT-REDACTION-001 — cross-sink parity (memory egress == audit const memoryValue = memoryGuard.redactDeep(GLUED_PAYLOAD()).value as Record; // Cross-sink equality — memory egress == audit sink for the glued credentials. expect(memoryValue).toEqual(auditDetails); - // The credential subspan is masked, the benign glued leading char (`key`/`x`/`id`) survives. + // The credential subspan is masked, the benign glued leading char (`key`/`x`/`id`/`aws`) survives. expect(memoryValue.hfref).toBe(`key${M}`); expect(memoryValue.gskref).toBe(`x${M}`); expect(memoryValue.glref).toBe(`id${M}`); + expect(memoryValue.ghpref).toBe(`key${M}`); // P1: glued GitHub classic ghp_ now caught in both sinks + expect(memoryValue.akiaref).toBe(`aws${M}`); // P1: glued AWS AKIA now caught in both sinks expect(memoryValue.note).toBe("just a note"); // Benign `…ghs_` identifiers survive byte-identical in BOTH sinks (no over-redaction), including // a CONTIGUOUS base62 run that only the leading `\b` closes. @@ -258,7 +266,7 @@ describe("SEC-EVENT-REDACTION-001 — cross-sink parity (memory egress == audit // No credential body survives in either sink's serialization. for (const sink of [auditDetails, memoryValue]) { const json = JSON.stringify(sink); - for (const cred of [HF_GLUED.slice(3), GSK_GLUED.slice(1), GLPAT_GLUED.slice(2)]) { + for (const cred of [HF_GLUED.slice(3), GSK_GLUED.slice(1), GLPAT_GLUED.slice(2), GHP_GLUED.slice(3), AKIA_GLUED.slice(3)]) { expect(json, cred).not.toContain(cred); } } From 61b2a6dba21f309f7b9cd8478669c74ee157b252 Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.8 (1M context)" Date: Sat, 18 Jul 2026 17:34:47 -0700 Subject: [PATCH 7/7] =?UTF-8?q?fix(security):=20AKIA=20de-bounding=20via?= =?UTF-8?q?=20context-sensitive=20negative-lookbehind=20=E2=80=94=20protec?= =?UTF-8?q?t=20SLOVAKIA/CZECHOSLOVAKIA=20all-caps=20constants=20(SEC-SECRE?= =?UTF-8?q?T-GLUED-PREFIX-001=20round-5=20NO-DEGRADE)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Advisor found a NO-DEGRADE regression in the round-4 plain de-`\b` AKIA branch: `AKIA` is the SUFFIX of the country names SLOVAKIA / CZECHOSLOVAKIA (proper nouns absent from /usr/share/dict/words, so the per-word dict analysis missed them). A plain leading-`\b` drop over-redacts an ALL-CAPS region constant — `SLOVAKIAREGIONCODE2024AB` -> `SLOV[REDACTED_SECRET]` — at the detector, realtime redactor, memory content/metadata, and audit JSONL. Fix (AKIA branch only): change `/AKIA[0-9A-Z]{16}\b/gu` to a context-sensitive negative-lookbehind `/(?`), NOT when embedded after an UPPERCASE letter or DIGIT (mid-all-caps-word `SLOVAKIA…`, mid-alphanumeric run `PROJECT2AKIA…`). The lookbehind is zero-width, so the redacted span is still exactly `AKIA<16>` (benign lead preserved). Robust — no per-word enumeration. GitHub {ghp,ghr}, the AWS keep-`\b` branch, and the 9 vendor prefixes are UNCHANGED (ghp/ghr are not substrings of any all-caps word; only AKIA is a country-word suffix). RED on 2c83630f (`SLOVAKIAREGIONCODE2024AB` corrupted); GREEN after (unchanged through detector + all 3 sinks). `keyAKIA<16>` + standalone/delimited AWS keys still caught at all sinks; the whole prior benign corpus stays green. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01JhCDTLngxAf6MDh4Bfcd48 --- src/security/friday-secret-shape-redactor.ts | 56 +++++++++-------- .../friday-event-payload-redactor.test.ts | 4 ++ ...friday-memory-output-filter-secret.test.ts | 5 +- .../friday-secret-shape-redactor.test.ts | 63 ++++++++++++++++--- ...nt-redaction-001-cross-sink-parity.test.ts | 6 ++ 5 files changed, 98 insertions(+), 36 deletions(-) diff --git a/src/security/friday-secret-shape-redactor.ts b/src/security/friday-secret-shape-redactor.ts index ae5bdf3c0..b5e5bfb3c 100644 --- a/src/security/friday-secret-shape-redactor.ts +++ b/src/security/friday-secret-shape-redactor.ts @@ -262,34 +262,40 @@ const SECRET_CONTENT_PATTERNS: readonly SecretContentPattern[] = [ // a shapeless 40-char base64 with no prefix — it is caught by its KEY NAME (`secretAccessKey`, // in the sensitive-field-name set), NOT a shape, so it is not (and must not be) a content pattern. // - // GLUED-PREFIX (SEC-SECRET-GLUED-PREFIX-001): the AWS family is SPLIT by prefix and the leading `\b` is - // DROPPED ONLY on `AKIA`. The AWS body is `[0-9A-Z]{16}` — UPPERCASE + DIGIT, i.e. the alphabet of an + // GLUED-PREFIX (SEC-SECRET-GLUED-PREFIX-001): the AWS family is SPLIT by prefix and only `AKIA` catches + // a glued credential. The AWS body is `[0-9A-Z]{16}` — UPPERCASE + DIGIT, i.e. the alphabet of an // all-caps constant / ULID / base32 blob — so (UNLIKE the mixed-case github body above) a member - // over-redacts ONLY when it sits inside an ALL-CAPS 20+ contiguous run; a lowercase prose word never - // collides. A member is de-`\b`'d ONLY if it fails ALL of: (a) a COMMON word-ending that forms all-caps - // runs, (b) ULID-constructible (Crockford base32 = 0-9 A-Z minus I,L,O,U — a member of only Crockford - // chars can sit mid-ULID), and (c) a common all-caps ACRONYM / fragment: - // • AKIA — has `I` → NOT ULID-constructible; only OBSCURE lowercase word-endings (aph-AKIA, - // leucopl-AKIA, Lat-AKIA — medical / geographic, never written as ALL-CAPS 20+ runs); no common - // all-caps acronym source → fails (a),(b),(c) → UNAMBIGUOUS → `\b` DROPPED (a glued `keyAKIA<16>` - // is now caught; this is the Advisor P1 canary). - // • ASIA — has `I` (not ULID) BUT a COMMON word-ending (Eur-ASIA, Austral-ASIA, aph-ASIA) written - // all-caps in region constants (…ASIA…, AUSTRALASIA…, EURASIA…) → fails (a),(c) → AMBIGUOUS → `\b` KEPT. + // over-redacts ONLY when it sits inside an ALL-CAPS / alphanumeric 20+ contiguous run; a lowercase prose + // word never collides. The 7 other prefixes KEEP `\b` — each is (a) a COMMON all-caps word-ending, (b) + // ULID-constructible (Crockford base32 = 0-9 A-Z minus I,L,O,U — a member of only Crockford chars can + // sit mid-ULID), or (c) a common all-caps ACRONYM / fragment: + // • ASIA — a COMMON word-ending (Eur-ASIA, Austral-ASIA, aph-ASIA) written all-caps in region + // constants (…ASIA…, AUSTRALASIA…, EURASIA…) → (a),(c) → AMBIGUOUS → `\b` KEPT. // • AGPA / ANPA / ANVA — Crockford-only (no I/L/O/U) → ULID-CONSTRUCTIBLE (sit mid-ULID, e.g. - // `012345AGPA…`); ANVA is also inside C-ANVA-S → fails (b) → AMBIGUOUS → `\b` KEPT. - // • AIDA — has `I` (not ULID) BUT a common all-caps ACRONYM (the AIDA marketing model, PROJECT-AIDA, - // the opera) → fails (c) → AMBIGUOUS → `\b` KEPT. - // • AROA — has `O` (not ULID) BUT the place name AOTE-AROA (all-caps-able in NZ constants) → fails - // (a),(c) → AMBIGUOUS → `\b` KEPT. - // • AIPA — has `I` (not ULID) BUT a prefix of the common all-caps acronym / org AIPA-C - // (`AIPACPOLICYCONFERENCE…`) → fails (c) → AMBIGUOUS → `\b` KEPT. - // Effect: a DELIMITED / standalone / labeled AWS key of ANY prefix is STILL caught (the common case — - // AWS keys are ~always delimited in env/config); a glued `keyAKIA<16>` is now REDACTED at exactly the - // credential subspan, while a glued benign all-caps / ULID carrying an AMBIGUOUS fragment - // (`012345AGPA…`, `…ASIA…WIDE…`, `PROJECTAIDA…`) is UNCHANGED (`AKIA` never appears in those blobs). + // `012345AGPA…`); ANVA is also inside C-ANVA-S → (b) → AMBIGUOUS → `\b` KEPT. + // • AIDA — a common all-caps ACRONYM (the AIDA marketing model, PROJECT-AIDA, the opera) → (c) → + // AMBIGUOUS → `\b` KEPT. + // • AROA — the place name AOTE-AROA (all-caps-able in NZ constants) → (a),(c) → AMBIGUOUS → `\b` KEPT. + // • AIPA — a prefix of the common all-caps acronym / org AIPA-C (`AIPACPOLICYCONFERENCE…`) → (c) → + // AMBIGUOUS → `\b` KEPT. + // • AKIA — has `I` → NOT ULID-constructible, and no common all-caps ACRONYM. BUT it IS the SUFFIX of + // the country names SLOV-AKIA and CZECHOSLOV-AKIA (proper nouns absent from /usr/share/dict/words, + // so the per-word dict analysis missed them) — an ALL-CAPS `SLOVAKIA<16>` region constant would be + // corrupted by a plain leading-`\b` drop. So AKIA is de-bounded with a CONTEXT-SENSITIVE + // NEGATIVE-LOOKBEHIND `(?` / `=AKIA<16>` / ` AKIA<16>`), NOT when it is embedded + // after an UPPERCASE letter or DIGIT (mid-all-caps-word `SLOVAKIA…` / mid-alphanumeric run + // `PROJECT2AKIA…`). The lookbehind is zero-width, so the redacted span is still exactly `AKIA<16>` + // and the benign leading char survives byte-for-byte. This is robust (no per-word enumeration). + // Effect: a DELIMITED / standalone / labeled / glued-after-lowercase AWS key is STILL caught (the common + // case — AWS keys are ~always delimited in env/config, and the glued `keyAKIA<16>` P1 canary is caught), + // while an ALL-CAPS / alphanumeric blob carrying an AMBIGUOUS fragment (`012345AGPA…`, `…ASIA…WIDE…`, + // `PROJECTAIDA…`) OR `SLOVAKIA…`/`CZECHOSLOVAKIA…`/`PROJECT2AKIA…` (AKIA after `[A-Z0-9]`) is UNCHANGED. { - // UNAMBIGUOUS — leading `\b` DROPPED so a glued `AKIA<16>` is caught. - pattern: /AKIA[0-9A-Z]{16}\b/gu, + // UNAMBIGUOUS — de-bounded via a CONTEXT-SENSITIVE negative-lookbehind (NOT a plain `\b`-drop) so a + // glued `AKIA<16>` is caught but a mid-all-caps-word `SLOVAKIA<16>` is NOT. + pattern: /(? { // AWS AKIA word-fragment: benign ULID / all-caps id with ASIA/AGPA glued after a word char. ulid: "012345AGPABCDEFGHJKMNPQRST", // pragma: allowlist secret — benign ULID (AKIA-family scanner false positive) caps: "AUSTRALASIAWIDEDEPLOYMENT01", // pragma: allowlist secret — benign all-caps constant (AKIA-family scanner false positive) + // AKIA is the SUFFIX of SLOV-AKIA / CZECHOSLOV-AKIA — the `(?` region constant UNCHANGED (a plain `\b`-drop corrupts it → `SLOV[REDACTED]`). + country: "SLOVAKIAREGIONCODE2024AB", // pragma: allowlist secret — AKIA after `V` (uppercase) + country2: "CZECHOSLOVAKIAREGIONCODE012345", // pragma: allowlist secret — AKIA after `V` }; const out = redactEventPayload(payload); expect(out).toEqual(payload); diff --git a/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts b/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts index 9c7ae6124..6da43482d 100644 --- a/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts +++ b/test/unit/memory/guard/services/friday-memory-output-filter-secret.test.ts @@ -345,7 +345,10 @@ describe("FridayMemoryOutputFilter — SEC-SECRET-GLUED-PREFIX-001 glued distinc it("NO-DEGRADE: benign `…ghs_` identifiers survive content / metadata / tags byte-identical", () => { const benign = ["walkthroughs_completed_counter", "walkthroughs_completedThisWeek", "coughs_e3b0c44298fc1c14", "breakthroughs_this_quarter_list", "laughs_per_minute_counter", // AWS AKIA word-fragment: benign ULID / all-caps id with ASIA/AGPA glued after a word char. - "012345AGPABCDEFGHJKMNPQRST", "AUSTRALASIAWIDEDEPLOYMENT01"]; // pragma: allowlist secret — benign ULID/all-caps (AKIA-family scanner false positives) + "012345AGPABCDEFGHJKMNPQRST", "AUSTRALASIAWIDEDEPLOYMENT01", // pragma: allowlist secret — benign ULID/all-caps (AKIA-family scanner false positives) + // AKIA is the SUFFIX of SLOV-AKIA / CZECHOSLOV-AKIA — the `(?` → `SLOV[REDACTED_SECRET]`). + "SLOVAKIAREGIONCODE2024AB", "CZECHOSLOVAKIAREGIONCODE012345"]; // pragma: allowlist secret — AKIA after uppercase `V` const out = filter.filterItem(makeItem({ content: "metric name walkthroughs_started_and_completed today", metadata: { walk: benign[0], breakt: benign[1], note: "keep" }, diff --git a/test/unit/security/friday-secret-shape-redactor.test.ts b/test/unit/security/friday-secret-shape-redactor.test.ts index ca3bdefd8..fc5e43e53 100644 --- a/test/unit/security/friday-secret-shape-redactor.test.ts +++ b/test/unit/security/friday-secret-shape-redactor.test.ts @@ -434,16 +434,16 @@ describe("friday-secret-shape-redactor", () => { ["SendGrid SG.<22>.<43>", SG_SECRET], // pragma: allowlist secret ["Square sq0atp-", seg("sq0atp-", "0123456789abcdefghijklABCDwxyz")], // pragma: allowlist secret ["Square sq0csp-", seg("sq0csp-", "0123456789abcdefghijklABCDwxyz")], // pragma: allowlist secret - // GitHub classic {ghp,ghr} and AWS {AKIA} were SPLIT out of their alternations and de-`\b`'d - // (SEC-SECRET-GLUED-PREFIX-001 P1 fix): `ghp`/`ghr` have ZERO dict word-endings; `AKIA` has `I` - // (not ULID-constructible), only obscure lowercase word-endings (aphakia/leucoplakia — never - // all-caps 20+ runs), and no common all-caps acronym → all three are provably NOT benign fragments, - // so a glued `keyghp_<36>` / `keyghr_<…>` / `keyAKIA<16>` real credential is now CAUGHT. The - // AMBIGUOUS members {gho,ghs,ghu,github_pat_} and {ASIA,AGPA,AIDA,AROA,AIPA,ANPA,ANVA} KEEP `\b` - // (word-ending / ULID / acronym fragments) — see KEPT. + // GitHub classic {ghp,ghr} were SPLIT out of the classic alternation and PLAIN de-`\b`'d + // (SEC-SECRET-GLUED-PREFIX-001 P1 fix): both have ZERO dict word-endings and are not a benign + // fragment, so a glued `keyghp_<36>` / `keyghr_<…>` (after ANY word char) is now CAUGHT. The + // AMBIGUOUS classic members {gho,ghs,ghu,github_pat_} KEEP `\b` — see KEPT. + // NB: AWS `AKIA` is ALSO de-bounded (P1 canary) but via a CONTEXT-SENSITIVE `(? { "AIPACPOLICYCONFERENCE2024ABC", // pragma: allowlist secret — AIPA is a prefix of the all-caps org AIPAC "AOTEAROANEWZEALANDGOVT01ABCD", // pragma: allowlist secret — AROA is inside the place name AOTEAROA "OPENCANVASRENDERINGCONTEXT2D", // pragma: allowlist secret — ANVA is inside CANVAS (also ULID-constructible) + // Negative controls for the de-`\b`'d AKIA member: `AKIA` is the SUFFIX of the country names + // SLOV-AKIA / CZECHOSLOV-AKIA (absent from the dict, so the per-word analysis missed them). The + // `(?` glued mid-all-caps-word / mid-alphanumeric-run + // UNCHANGED (would corrupt under a plain leading-`\b` drop — `SLOVAKIA…` → `SLOV[REDACTED]`). + "SLOVAKIAREGIONCODE2024AB", // pragma: allowlist secret — AKIA after `V` (uppercase) → not a credential + "CZECHOSLOVAKIAREGIONCODE012345", // pragma: allowlist secret — AKIA after `V` → not a credential + "deployed to SLOVAKIAREGIONCODE2024AB today", // pragma: allowlist secret — free-text SLOVAKIA embedding + "PROJECT2AKIAXYZ0123456789ABC", // pragma: allowlist secret — AKIA after `2` (digit) → not a credential "9f8e7d6c5b4a3928170695f4e3d2c1b0", // pragma: allowlist secret — 32-hex id / git blob "/var/log/hf_service/npm_cache/output.log", // file path with hf_/npm_ short segments "GOCSPX_notasecret_underscore", // GOCSPX_ (underscore, not the required hyphen) @@ -607,8 +615,10 @@ describe("friday-secret-shape-redactor", () => { }); // AWS sensitivity boundary (SEC-SECRET-GLUED-PREFIX-001 P1 split): the AWS alternation is SPLIT by - // prefix. `AKIA` (has `I` → not ULID; only obscure lowercase word-endings; no common all-caps acronym) - // is de-`\b`'d → a glued `keyAKIA<16>` is now CAUGHT (P1 canary). The AMBIGUOUS members + // prefix. `AKIA` (has `I` → not ULID; no common all-caps acronym) is de-bounded via a context-sensitive + // `(?` + // is CAUGHT (P1 canary) while a mid-all-caps-word `SLOVAKIA<16>` is NOT. The AMBIGUOUS members // {ASIA,AGPA,AIDA,AROA,AIPA,ANPA,ANVA} — word-endings (Eur-ASIA), ULID-constructible (AGPA/ANPA/ANVA), // or acronym fragments (AIDA, AIPA→AIPAC, AROA→AOTEAROA) — KEEP `\b`, so a benign all-caps / ULID id // where the fragment is GLUED after a word char is UNCHANGED. A DELIMITED / standalone / labeled AWS @@ -637,5 +647,38 @@ describe("friday-secret-shape-redactor", () => { expect(findSecretShapeSpans(seg("x", K)), `${p} glued span`).toEqual([]); } }); + + // AWS AKIA context-lookbehind (SEC-SECRET-GLUED-PREFIX-001 round-5 NO-DEGRADE): `AKIA` is the SUFFIX of + // the country names SLOV-AKIA / CZECHOSLOV-AKIA (proper nouns absent from /usr/share/dict/words, so the + // per-word dict analysis missed them). A plain leading-`\b` drop corrupts an ALL-CAPS `SLOVAKIA<16>` + // region constant (`SLOVAKIA<16>` → `SLOV[REDACTED]`). The `(? { + const K = seg("AKIA", "IOSFODNN7EXAMPLE"); // pragma: allowlist secret — AKIA + 16 [0-9A-Z] + // NEGATIVE: AKIA preceded by an UPPERCASE letter or DIGIT → NOT a credential → UNCHANGED, 0 spans. + const negatives = [ + "SLOVAKIAREGIONCODE2024AB", // pragma: allowlist secret — AKIA after `V` + "CZECHOSLOVAKIAREGIONCODE012345", // pragma: allowlist secret — AKIA after `V` + "deployed to SLOVAKIAREGIONCODE2024AB today", // pragma: allowlist secret — free-text SLOVAKIA embedding + seg("PROJECT2", K), // AKIA after digit `2` + seg("XAKIA", "IOSFODNN7EXAMPLE"), // pragma: allowlist secret — AKIA after uppercase `X` (mid-all-caps-word) + ]; + for (const n of negatives) { + expect(redactSecretShapesInString(n), n).toBe(n); + expect(findSecretShapeSpans(n), n).toEqual([]); + } + // POSITIVE: AKIA starting a fresh token (lowercase / symbol / space / assignment / string start) → CAUGHT. + expect(redactSecretShapesInString(seg("key", K))).toBe(`key${M}`); // lowercase glue (P1 canary) + expect(redactSecretShapesInString(seg("aws", K))).toBe(`aws${M}`); // lowercase glue + expect(redactSecretShapesInString(`(${K})`)).toBe(`(${M})`); // symbol glue + expect(redactSecretShapesInString(`region ${K}`)).toBe(`region ${M}`); // whitespace delimited + expect(redactSecretShapesInString(`aws_key=${K}`)).toBe(`aws_key=${M}`); // assignment + expect(redactSecretShapesInString(K)).toBe(M); // string start + // Span still excludes the benign lowercase lead (lookbehind is zero-width). + const spans = findSecretShapeSpans(seg("key", K)); + expect(spans.length).toBe(1); + expect(seg("key", K).slice(spans[0]!.start, spans[0]!.end)).toBe(K); + }); }); }); diff --git a/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts b/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts index 1b2ee7903..66c2964c7 100644 --- a/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts +++ b/test/unit/security/sec-event-redaction-001-cross-sink-parity.test.ts @@ -240,6 +240,10 @@ describe("SEC-EVENT-REDACTION-001 — cross-sink parity (memory egress == audit // survive in BOTH sinks — the AKIA branch keeps `\b` (was corrupted while AKIA was de-`\b`'d). ulid: "012345AGPABCDEFGHJKMNPQRST", // pragma: allowlist secret — benign ULID (AKIA-family scanner false positive) caps: "AUSTRALASIAWIDEDEPLOYMENT01", // pragma: allowlist secret — benign all-caps constant (AKIA-family scanner false positive) + // AKIA is the SUFFIX of SLOV-AKIA / CZECHOSLOV-AKIA — the `(?`). + country: "SLOVAKIAREGIONCODE2024AB", // pragma: allowlist secret — AKIA after uppercase `V` + country2: "CZECHOSLOVAKIAREGIONCODE012345", // pragma: allowlist secret — AKIA after uppercase `V` }); it("glued distinctive-prefix credentials redact identically in the AUDIT sink and MEMORY redactDeep", async () => { @@ -263,6 +267,8 @@ describe("SEC-EVENT-REDACTION-001 — cross-sink parity (memory egress == audit expect(memoryValue.cough).toBe("coughs_e3b0c44298fc1c14"); expect(memoryValue.ulid).toBe("012345AGPABCDEFGHJKMNPQRST"); // pragma: allowlist secret — benign ULID (AKIA-family scanner false positive) expect(memoryValue.caps).toBe("AUSTRALASIAWIDEDEPLOYMENT01"); // pragma: allowlist secret — benign all-caps constant (AKIA-family scanner false positive) + expect(memoryValue.country).toBe("SLOVAKIAREGIONCODE2024AB"); // pragma: allowlist secret — SLOVAKIA suffix preserved by the lookbehind in both sinks + expect(memoryValue.country2).toBe("CZECHOSLOVAKIAREGIONCODE012345"); // pragma: allowlist secret — CZECHOSLOVAKIA suffix preserved // No credential body survives in either sink's serialization. for (const sink of [auditDetails, memoryValue]) { const json = JSON.stringify(sink);