diff --git a/CHANGELOG.md b/CHANGELOG.md index e6bfef4..ca0dab6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,137 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Versioning: [SemVer](https://semver.org/spec/v2.0.0.html). +## [0.4.0] — bench correctness + detection fixes + +Started as gateway hardening from a Claude Code proxy integration +report. Two of the four filters that report motivated did not survive +measurement and were dropped; benchmarking them surfaced a defect in +the benchmark itself that had been mis-ranking every tool in the +published table. Public API surface unchanged. + +### Behavior changes + +- **Placeholder-escape PUA sentinels stripped before detection** + (#43). User-authored `{{...}}` templates wrapped in PUA sentinels + (`U+E000` / `U+E001`) used to be tagged as `private_person` by GLiNER + when `any-ascii` left the codepoints in the normalized text. Vault + substitution then corrupted the user's `{{...}}` into + `{{PII_PRIVATE_PERSON_*}}...{{`. Now stripped in + `normalizeForDetection` via the existing `ZERO_WIDTH_CHARS` set. +- **`private_geolocation` gated on coordinate shape.** The schema + defines the label as a coordinate — all three recognizers that own it + match lat/lon literals, and `core:geo-latlon-decimal` range-validates. + The GLiNER head is prompted with the label zero-shot and never trained + on it, so it was applying it to place names, regions and postcodes: + only **7.6%** of its emissions on `ai4privacy-300k-heldout` were + coordinate-shaped, the rest `Deutschland`, `Sachsen`, `52396`. Every + gold span of the class is a lat/lon pair, so the gate removes false + positives without touching a conforming true positive — measured + **−2,317 FP, ±0 TP**, precision 0.129 → 0.902, recall unchanged. Runs + before cross-label dedupe, so the model's competing `private_address` + candidate survives the overlap instead of losing it to a label it + should never have won (+83 recovered TP: `private_ip` +60, + `private_address` +11, `private_vehicle_id` +7, `private_mac` +5). +- **Public-URL allowlist on by default** (#45). URLs targeting + `PUBLIC_URL_HOSTS` (github / gitlab / docs.python.org / anthropic / + openai / huggingface / wikipedia / mdn / stackoverflow / cloud + vendor docs / standards bodies — full set in `src/url-filter.ts`) + are dropped from `private_url` output. Subdomain match + (`docs.python.org` covered by `python.org`); `www.` stripped. Opt + out with `new NullPii({ urlAllowlist: 'none' })`. + + An allowlisted URL is dropped **only when it carries no other PII + span**. `core:url` matches `[^\s<>"]+`, so PII glued to a URL is + swallowed into the URL span and containment-elimination then drops + the inner span regardless of score; dropping the outer span would + have emitted the nested secret as plaintext behind a 20-character + allowlisted prefix (`https://github.com/a,AKIA…`). Fail-safe by + construction: a reference URL embedding anything identifying reverts + to whole-URL redaction. Also refuses to judge a span carrying more + than one scheme (`core:url` does not stop at `,`), and matches on + `hostname` rather than `host` so a non-default port no longer + silently disables the allowlist. + +### Dropped before release + +- **Template syntax mask** (#44) — **closed, not shipped.** #43 already + fixes the motivating case: ten template-variable probes on `main`, + including the two this feature cited, produce zero tagged spans, and + the regression test in `test/nullpii.test.ts` passes without it. The + implementation also made `MAX_INPUT_BYTES`-sized input quadratic + (69.7 s on 1 MB, against ~70 ms for the whole rest of the pipeline), + and unbalanced delimiters suppressed redaction in ordinary text — a C + array initializer and a one-character typo both leaked real emails. +- **`private_date` threshold 0.85** (#46) — **closed, not shipped.** It + does not reach its own target: gateway boilerplate dates score + 0.993–1.000, so an 0.85 cut removes none of them. The score tracks + span-boundary confidence, not PII relevance — a birth date inside a + two-date span scores 0.574 and a neutral one scores 0.609 — so no cut + point separates them. Measured cost was **419 true positives dropped + to remove 453 false ones** (1.08 : 1), with `private_date` F1 going + *down* 0.004. What it did remove was boundary fragments (`'62'`, a + bare `'1938'` split off `March 17, 1938`), not the copyright footers + it was aimed at. + +### Bench + +The benchmark's ai4privacy and isotonic loaders resolved unknown +upstream labels through `dict.get()`, which returns `None` both for +"deliberately excluded" and for "never seen". Gold was deleted with no +signal — **56.4%** on `ai4privacy-300k-heldout`, ~11% on the isotonic +`-heldout` rows. Because `macro_f1` skips zero-support classes, +predictions on a class whose gold had been deleted cost nothing, so the +bug hid recall *and* inflated precision. All 25 recovered labels are +aliases of keys already in the maps; mapping and exclusion are now +distinct and an unknown label raises. + +This moves every tool, not just `nullpii` — on `ai4privacy-300k-heldout` +it reorders 8 of 9 (`piiranha` +0.2347, `gliner-pii-large-v1` +0.1801, +`gliner-onnx-pii-fp32` +0.1423, `nullpii` −0.0519, `presidio` −0.1746). +The published v0.3.0 column is therefore **not comparable** to the +v0.4.0 column; the middle column restates v0.3.0 on the corrected +metric, and only the last delta is a product improvement. + +| Metric | v0.3.0 as published | v0.3.0 re-scored | v0.4.0 | Δ (product) | +|---|---:|---:|---:|---:| +| OOD-7 macro F1 | 0.7784 | 0.8043 | **0.8290** | **+0.0247** | +| `nullpii-bench` F1 | 0.4228 | 0.4228 | **0.4519** | **+0.0291** | +| Cold start (M5 Pro CPU) | ~756 ms | — | not re-measured | — | + +`presidio-synthetic` moves by exactly `+0.0000` — it carries no +`private_geolocation` gold and none of its 89 URLs are allowlisted, +which makes it the negative control for both behavior changes. + +The 11 canonical rows the gold fix did not touch still carry v0.3.0 +numbers; the published table is not fully regenerated yet. + +### Config additions + +- `NullPiiConfig.urlAllowlist?: 'none'` — opt out of public-URL allowlist. + +### Internal + +- `src/geo-filter.ts` — `isCoordinateShaped`, `dropNonCoordinateGeolocation`. Range and null-island rejection defer to the existing `latLonPairInRange`. +- `src/url-filter.ts` — `PUBLIC_URL_HOSTS`, `isPublicUrl`, `dropCleanPublicUrlSpans`. +- `src/placeholder-escape.ts` — sentinel chars now exported individually as `PLACEHOLDER_SENTINEL_LEFT` / `PLACEHOLDER_SENTINEL_RIGHT` for reuse in `normalize.ts`. Source uses `''` / `''` escape syntax instead of raw PUA (grep-friendly, no mojibake on GitHub mobile). +- `packages/eval/scripts/ood_macro.py` — single definition of the OOD-7 set, computed from any `matrix.json`. The root README enumerated 7 datasets and `packages/eval/datasets/README.md` said 5, a 0.0128 discrepancy that would have silently blocked any regression gate. Exits non-zero if a cell is missing or `CRASHED`. +- `packages/eval/pyproject.toml` — the wheel force-included the whole `datasets/` directory, so any file dropped there was redistributed. Now a per-file allowlist with the upstream licence annotated per line. + +### Known issues + +- Partially-overlapping spans corrupt the output and break the restore + round-trip. `x https://github.com/users/john.doe@acme.com y` yields + `private_url[2,44]` and `private_email[27,46]` — IoU 0.39, below the + dedupe threshold, so both survive and overwrite each other during + vault substitution; the email span also ends past the end of the + string. Pre-existing in 0.3.0, not introduced here. + +### Test plan + +- 300 unit + integration tests passing (was 271 on v0.3.0). +- New tests: `test/geo-filter.test.ts` (10 — coordinate shapes, the place names and postcodes the head mislabels, IPv4 not reading as a pair, range and null-island rejection), `test/url-filter-nested.test.ts` (10 — nested-secret retention, concatenated URLs, non-default ports, near-miss hosts), `test/url-filter.test.ts` (7), plus integration cases in `test/nullpii.test.ts`. +- `test/url-filter.test.ts`'s span fixture was corrected: it parked every span at offset 0, which under a containment-aware drop rule reads as "this URL has PII nested inside it". + ## [0.3.0] — first public release Initial Apache-2.0 public release. Published packages: diff --git a/README.md b/README.md index 616cc0d..19c36ac 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ Two `nullpii` rows + one upstream-GLiNER row let readers isolate the model from - **`gliner-onnx-pii-fp32`** — the unmodified upstream [`onnx-community/gliner_multi_pii-v1`](https://huggingface.co/onnx-community/gliner_multi_pii-v1) ONNX, same bare consumer. Baseline before any project fine-tuning. - **`nullpii`** — the npm package (full runtime): published model + recognizer pack + adversarial preprocessor + base64 decoder + reversible vault. -v0.3.0 bench (M5 Pro CPU, 2026-05-18 + opf 2026-05-20, full 9×16 matrix). OOD macro for `nullpii` = **0.7784** (presidio-synthetic + isotonic-{en,de,fr,it}-heldout + ai4privacy-300k-heldout + tab-echr). +v0.3.0 bench (M5 Pro CPU, 2026-05-18 + opf 2026-05-20, full 9×16 matrix). OOD macro for `nullpii` = **0.7784** over the **OOD-7** set (presidio-synthetic + isotonic-{en,de,fr,it}-heldout + ai4privacy-300k-heldout + tab-echr). The set is defined once, in `packages/eval/scripts/ood_macro.py`; recompute the headline from any `matrix.json` with `python packages/eval/scripts/ood_macro.py ` rather than restating it by hand. | Dataset | n | **`nullpii`** | **`nullpii-bare`** | `nemotron-pii-raw` | `gliner-pii-large-v1` | `gliner-onnx-pii-fp32` | `deberta` | `piiranha` | `presidio` | `opf` | |---|---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| @@ -174,7 +174,8 @@ v0.3.0 bench (M5 Pro CPU, 2026-05-18 + opf 2026-05-20, full 9×16 matrix). OOD m Legend: - **bold** = best F1 in the row - ⚠ = the dataset overlaps the training distribution of at least one competitor in the row — read those cells with caution -- ⚐ = in-distribution for `nullpii` itself — regression cell, **not** counted in the OOD headline. The held-out OOD macro (0.7784) is computed over `presidio-synthetic` + `isotonic-{en,de,fr,it}-heldout` + `ai4privacy-300k-heldout` + `tab-echr` only. The `nullpii-internal-bench` row sits at the bottom of the table and is shown only as a regression watcher across releases — read it that way. +- ⚐ = in-distribution for `nullpii` itself — regression cell, **not** counted in the OOD headline. The held-out OOD macro (0.7784) is computed over the OOD-7 set only: `presidio-synthetic` + `isotonic-{en,de,fr,it}-heldout` + `ai4privacy-300k-heldout` + `tab-echr`. The `nullpii-internal-bench` row sits at the bottom of the table and is shown only as a regression watcher across releases — read it that way. +- the `-heldout` suffix means rows sliced **above `nullpii`'s own training offsets** (`_AI4_HELDOUT_OFFSET` / `_ISOTONIC_HELDOUT_ROW_OFFSET` in `packages/eval/scripts/bench_full.py`). It is a guarantee about `nullpii` and about no other tool in the row — a competitor trained on the same upstream corpus sees those rows as ordinary training data. - ‡ = competitor benched on its own training distribution (best-case self-report) - § = Presidio benched on its own evaluator dataset (best-case self-report) diff --git a/packages/eval/datasets/README.md b/packages/eval/datasets/README.md index 510bf7b..7f141d5 100644 --- a/packages/eval/datasets/README.md +++ b/packages/eval/datasets/README.md @@ -10,8 +10,14 @@ externally-licensed upstream splits. | File | Rows | Used in canonical bench | Origin | |---|---:|:---:|---| | `nullpii-bench.jsonl` | 2,361 | ✅ project-authored, see subsets below | self-authored, Apache-2.0 | -| `presidio-synthetic.jsonl` | 5,000 | ✅ external (held-out OOD-5) | Microsoft Presidio seed, MIT | -| `tab-echr-test.jsonl` | 127 | ✅ EU legal test split | TAB ECHR test (ACL 2022), MIT | +| `presidio-synthetic.jsonl` | 5,000 | ✅ external, member of **OOD-7** | Microsoft Presidio seed, MIT | +| `tab-echr-test.jsonl` | 127 | ✅ EU legal test split, member of **OOD-7** | TAB ECHR test (ACL 2022), MIT | + +Every file listed here is redistributed inside the `nullpii-eval` wheel via +the per-file allowlist in [`../pyproject.toml`](../pyproject.toml). A file +dropped into this directory does **not** ship until it is added there — +which is the point: adding the line forces a licence check first. Only +MIT / Apache-2.0 / BSD / ISC / CC0 upstreams are eligible. Additional bench rows (`ai4privacy-*`, `isotonic-*`, `nemotron-pii-*`, `argilla-pii`) are fetched from HuggingFace at bench time by the per-tool @@ -64,10 +70,38 @@ re-fetching the upstream pool from internal storage. ⚠ `nullpii-bench` is in-distribution for the project pipeline. F1 on this dataset is a regression test for the runtime (preprocessor, -recognizer pack, base64 decoder), not an OOD generalisation claim. The -held-out OOD headline is the macro over 5 external datasets -(`presidio-synthetic` + `isotonic-{en,de,fr,it}-heldout`) reported in -the top-level README. +recognizer pack, base64 decoder), not an OOD generalisation claim. + +### OOD-7 — the held-out headline set + +The OOD headline reported in the top-level README is the macro over +**seven** external datasets, named **OOD-7** here and in the root README: + +``` +presidio-synthetic +isotonic-en-heldout isotonic-de-heldout +isotonic-fr-heldout isotonic-it-heldout +ai4privacy-300k-heldout +tab-echr +``` + +Membership criterion: the dataset is externally authored **and** no part +of it entered nullpii's training distribution. The `-heldout` suffix means +rows sliced above nullpii's own training offsets +(`_AI4_HELDOUT_OFFSET` / `_ISOTONIC_HELDOUT_ROW_OFFSET`, +[`../scripts/bench_full.py`](../scripts/bench_full.py)) — it carries **no** +held-out guarantee for any other tool in the matrix. + +Do not restate this number by hand. Compute it from `matrix.json`: + +```bash +python packages/eval/scripts/ood_macro.py packages/eval/published-bench/matrix.json +``` + +An earlier revision of this file described the headline as a macro over +five datasets; that was wrong and disagreed with the root README by +0.0128 F1 (OOD-5 = 0.7656 vs OOD-7 = 0.7784 at v0.3.0). The root README +figure was the correct one. ## `presidio-synthetic` diff --git a/packages/eval/pyproject.toml b/packages/eval/pyproject.toml index b459ddc..2644074 100644 --- a/packages/eval/pyproject.toml +++ b/packages/eval/pyproject.toml @@ -22,5 +22,15 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["src/nullpii_eval"] +# Per-file allowlist, NOT a directory include. Anything under `datasets/` +# that is not named here stays out of the distributed Apache-2.0 wheel. +# Adding a line is a deliberate act that requires checking the upstream +# licence first: this directory is packaged and redistributed, so a file +# with incompatible terms (anything outside MIT/Apache-2.0/BSD/ISC/CC0) +# must never be listed. Bench rows fetched from HuggingFace at run time +# are unaffected — they are never written here. [tool.hatch.build.targets.wheel.force-include] -"datasets" = "nullpii_eval/datasets" +"datasets/README.md" = "nullpii_eval/datasets/README.md" +"datasets/nullpii-bench.jsonl" = "nullpii_eval/datasets/nullpii-bench.jsonl" # Apache-2.0, project-authored +"datasets/presidio-synthetic.jsonl" = "nullpii_eval/datasets/presidio-synthetic.jsonl" # MIT, Microsoft Presidio +"datasets/tab-echr-test.jsonl" = "nullpii_eval/datasets/tab-echr-test.jsonl" # MIT, TAB ECHR (ACL 2022) diff --git a/packages/eval/scripts/ood_macro.py b/packages/eval/scripts/ood_macro.py new file mode 100644 index 0000000..53ac20a --- /dev/null +++ b/packages/eval/scripts/ood_macro.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Compute the OOD-7 headline macro-F1 from a bench `matrix.json`. + +The held-out OOD headline quoted in the top-level README used to be a +hand-maintained number. Nothing computed it, so the root README and +`packages/eval/datasets/README.md` drifted apart: the root enumerated +seven datasets (0.7784) while the datasets README claimed five (0.7656), +a 0.0128 gap that silently invalidated any regression gate measured +against the wrong baseline. + +This script is the single source of truth. `OOD_7` below is the set; +change it here and nowhere else. + +Membership criterion: externally authored AND no part of it entered +nullpii's training distribution. The `-heldout` suffix means rows sliced +above nullpii's own training offsets (`_AI4_HELDOUT_OFFSET` / +`_ISOTONIC_HELDOUT_ROW_OFFSET` in `bench_full.py`) — it carries no +held-out guarantee for any other tool in the matrix. + +Usage: + python ood_macro.py packages/eval/published-bench/matrix.json + python ood_macro.py matrix.json --tool nullpii-bare + python ood_macro.py a/matrix.json --baseline b/matrix.json # A/B delta + +Exits non-zero if any OOD-7 cell is missing or did not complete, so a +truncated bench run can never be reported as a headline. +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +# The held-out OOD headline set. Order is the reporting order. +OOD_7 = ( + "presidio-synthetic", + "isotonic-en-heldout", + "isotonic-de-heldout", + "isotonic-fr-heldout", + "isotonic-it-heldout", + "ai4privacy-300k-heldout", + "tab-echr", +) + +_OK_STATUS = "OK" + + +def ood_cells(matrix: dict, tool: str) -> dict[str, float]: + """Per-dataset F1 for `tool` across OOD-7. Raises if any cell is unusable.""" + cells: dict[str, float] = {} + missing: list[str] = [] + for key in OOD_7: + cell = matrix.get(key, {}).get(tool) + if cell is None: + missing.append(f"{key}: absent") + continue + status = cell.get("status", _OK_STATUS) + if status != _OK_STATUS: + missing.append(f"{key}: status={status}") + continue + f1 = cell.get("f1") + if f1 is None: + missing.append(f"{key}: no f1") + continue + cells[key] = float(f1) + if missing: + raise SystemExit( + f"OOD-7 incomplete for tool '{tool}' — refusing to report a headline:\n " + + "\n ".join(missing) + ) + return cells + + +def ood_macro(matrix: dict, tool: str) -> float: + """Unweighted mean of the seven OOD-7 per-dataset macro-F1 scores.""" + cells = ood_cells(matrix, tool) + return sum(cells.values()) / len(cells) + + +def _load(path: Path) -> dict: + if not path.exists(): + raise SystemExit(f"no such matrix: {path}") + return json.loads(path.read_text()) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("matrix", type=Path, help="path to matrix.json") + ap.add_argument("--tool", default="nullpii", help="tool column (default: nullpii)") + ap.add_argument("--baseline", type=Path, default=None, + help="second matrix.json to diff against (A/B mode)") + args = ap.parse_args() + + cells = ood_cells(_load(args.matrix), args.tool) + macro = sum(cells.values()) / len(cells) + + if args.baseline is None: + for key, f1 in cells.items(): + print(f" {key:26} {f1:.4f}") + print(f"\nOOD-7 macro [{args.tool}] = {macro:.4f} (n={len(cells)} datasets)") + return + + base = ood_cells(_load(args.baseline), args.tool) + base_macro = sum(base.values()) / len(base) + print(f" {'dataset':26} {'baseline':>9} {'candidate':>10} {'delta':>9}") + for key in OOD_7: + delta = cells[key] - base[key] + flag = " " if abs(delta) < 5e-4 else (" +" if delta > 0 else " -") + print(f" {key:26} {base[key]:9.4f} {cells[key]:10.4f} {delta:+9.4f}{flag}") + delta = macro - base_macro + print(f"\n {'OOD-7 macro':26} {base_macro:9.4f} {macro:10.4f} {delta:+9.4f}") + if delta < 0: + print("\nREGRESSION — candidate is worse than baseline.", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/packages/eval/src/nullpii_eval/public_datasets.py b/packages/eval/src/nullpii_eval/public_datasets.py index 334a4ef..b9bb56c 100644 --- a/packages/eval/src/nullpii_eval/public_datasets.py +++ b/packages/eval/src/nullpii_eval/public_datasets.py @@ -242,11 +242,75 @@ def _load_ai4privacy( "APIKEY": "secret", "PASSWORD": "secret", "PIN": "secret", + # ─── pii-masking-300k vocabulary ────────────────────────────── + # Everything above was written against the 400k release. The 300k + # release spells the SAME taxonomy differently, so these 15 keys are + # aliases of keys already in this dict — not new categories. Without + # them 56.4% of 300k gold was deleted before scoring. Counts are + # spans recovered on `train[100000:105000]` (the -heldout row). + "GIVENNAME1": "private_person", # 1,358 — alias of GIVENNAME + "GIVENNAME2": "private_person", # 379 + "LASTNAME1": "private_person", # 1,489 — alias of LASTNAME + "LASTNAME2": "private_person", # 363 + "LASTNAME3": "private_person", # 122 + "TEL": "private_phone", # 1,447 — alias of TELEPHONENUM + "BOD": "private_date", # 1,501 — birth date, alias of DOB + "BUILDING": "private_address", # 1,196 — alias of BUILDINGNUM + "SECADDRESS": "private_address", # 475 — alias of SECONDARYADDRESS + "SOCIALNUMBER": "account_number", # 1,687 — alias of SOCIALNUM + "IDCARD": "account_number", # 1,592 — alias of IDCARDNUM + "PASSPORT": "private_passport", # 1,506 — alias of PASSPORTNUM + "DRIVERLICENSE": "private_driver_license", # 1,578 — alias of DRIVERLICENSENUM + "PASS": "secret", # 1,136 — password, alias of PASSWORD + "GEOCOORD": "private_geolocation", # 140 — alias of GPSCOORDINATES } +# Upstream labels with no home in nullpii's 14-label schema. Listing a +# label here is a DECISION ("this is not PII we detect"); leaving it out +# of both this set and `_AI4PRIVACY_LABELS` is a BUG. Keeping the two +# apart is the whole point — `.get()` used to return None for both cases, +# so an unrecognised upstream label was indistinguishable from a +# deliberate exclusion and silently deleted the gold span. +_AI4PRIVACY_IGNORED = frozenset({ + "SEX", # gender — no demographic class in the schema + "CARDISSUER", # card brand ("Diners Club International") — an org + # Honorific / rank held as a STANDALONE span ("Mr", "Papst", + # "Bürgermeisterin" — 224 distinct values, many occupational). Every + # other gold set in this bench folds the honorific INSIDE the person + # span (tab-echr: 842 of 1,063 person spans start "Mr/Mrs/Dr/…"), so + # mapping this would invent a span shape no other row uses. Adopting + # it is worth +0.0026 macro on ai4privacy-300k-heldout — declined on + # principle, not on score. + "TITLE", +}) + + def _map_ai4privacy_label(label: str) -> str | None: - return _AI4PRIVACY_LABELS.get(label.upper()) + """ai4privacy raw label → nullpii category, or None if deliberately ignored. + + Raises on an unknown label rather than dropping it. The 300k and 400k + releases ship DIFFERENT vocabularies for the same taxonomy (300k + `GIVENNAME1`/`LASTNAME1`/`TEL`/`BOD`/`PASS` vs 400k + `GIVENNAME`/`SURNAME`/`TELEPHONENUM`/`DOB`/`PASSWORD`), and this map + was authored against 400k only: 56.4% of 300k gold was being deleted + by variants absent from the dict — 15 of them, every one an alias of + a key already present. Silent drops also LAUNDER PRECISION, because + `macro_f1` skips zero-support classes: predictions on a class whose + gold was deleted cost nothing at all. + """ + key = label.upper() + mapped = _AI4PRIVACY_LABELS.get(key) + if mapped is not None: + return mapped + if key in _AI4PRIVACY_IGNORED: + return None + raise ValueError( + f"[ai4privacy] unknown gold label {label!r}. Add it to " + f"_AI4PRIVACY_LABELS (it is PII we detect) or to " + f"_AI4PRIVACY_IGNORED (it is not) — never leave it to fall " + f"through, that silently deletes gold and inflates precision.", + ) def _load_presidio_synthetic(max_samples: int | None) -> PublicDataset: @@ -348,7 +412,6 @@ def _map_presidio_entity_to_nullpii(entity_type: str) -> str | None: "LASTNAME": "private_person", "FULLNAME": "private_person", "PREFIX": "private_person", - "JOBTITLE": None, "EMAIL": "private_email", "PHONE_NUMBER": "private_phone", "PHONENUMBER": "private_phone", @@ -363,12 +426,10 @@ def _map_presidio_entity_to_nullpii(entity_type: str) -> str | None: "DATE": "private_date", "TIME": "private_date", "URL": "private_url", - "USERNAME": None, "ACCOUNT_NUMBER": "account_number", "ACCOUNTNUMBER": "account_number", "IBAN": "account_number", "CREDITCARDNUMBER": "account_number", - "CREDITCARDISSUER": None, "CREDITCARDCVV": "account_number", "SSN": "account_number", "PASSWORD": "secret", @@ -390,8 +451,73 @@ def _map_presidio_entity_to_nullpii(entity_type: str) -> str | None: "MACADDRESS": "private_mac", "NEARBYGPSCOORDINATE": "private_geolocation", "GPSCOORDINATES": "private_geolocation", + # ─── Parity with `_AI4PRIVACY_LABELS` ───────────────────────── + # Isotonic is an open mirror of ai4privacy (see `_load_isotonic` + # docstring), so the two maps describe ONE taxonomy. These keys were + # mapped on the three ai4privacy rows and silently dropped on the + # eight isotonic rows — the same gold label counted on some rows of + # the published matrix and invisible on others. Counts are spans + # recovered across the four `-heldout` slices. + "MIDDLENAME": "private_person", # 599 + "USERNAME": "private_person", # 601 — GDPR Art.4 online identifier; + # ai4privacy and argilla both map it + # to private_person. Was an explicit + # `None` here: same dataset family, + # opposite decisions. + "COUNTY": "private_address", # 571 + "SECONDARYADDRESS": "private_address", # 513 + "MASKEDNUMBER": "account_number", # 505 — 16-digit PANs + "BIC": "account_number", # 163 + "LITECOINADDRESS": "account_number", # 155 — sibling of the already-mapped + # BITCOINADDRESS / ETHEREUMADDRESS. + # Score-NEGATIVE for nullpii + # (−0.0013 macro): kept because the + # rule is schema membership, not score. + "PHONEIMEI": "account_number", # 474 — device id; `account_number` is + # this schema's bucket for structured + # registry numbers, cf. + # `core:device-serial-context`. + "IP": "private_ip", # 468 + "MAC": "private_mac", # 236 } +# No home in nullpii's 14-label schema. Same contract as +# `_AI4PRIVACY_IGNORED`: listed here = decided, absent from both = bug. +# NB: this set was chosen by schema membership, then checked SYMMETRICALLY +# — every entry was also scored to make sure the list is not just the +# mappings that happen to flatter nullpii. Three of them would LOWER our +# macro if mapped (AGE −0.0054, USERAGENT −0.0177, ACCOUNTNAME −0.0046) +# and are excluded anyway, on the same rule that keeps LITECOINADDRESS in. +_ISOTONIC_IGNORED = frozenset({ + "JOBTITLE", "JOBAREA", "JOBTYPE", "COMPANYNAME", # employment / org + "SEX", "GENDER", "AGE", "EYECOLOR", "HEIGHT", # demographic attributes + "AMOUNT", "CURRENCY", "CURRENCYCODE", # money identifies nobody + "CURRENCYNAME", "CURRENCYSYMBOL", + "CREDITCARDISSUER", # card brand — an org + "ACCOUNTNAME", # label of an account, not its number + "USERAGENT", # client string, not a secret + "ORDINALDIRECTION", # "Southwest" — not an address +}) + + +def _map_isotonic_label(raw_label: str) -> str | None: + """Isotonic raw label (`CITY_1`) → nullpii category, or None if ignored. + + Raises on an unknown label instead of dropping it, for the reason + given on `_map_ai4privacy_label`. + """ + key = _strip_index(raw_label).upper() + mapped = _ISOTONIC_LABEL_MAP.get(key) + if mapped is not None: + return mapped + if key in _ISOTONIC_IGNORED: + return None + raise ValueError( + f"[isotonic] unknown gold label {raw_label!r}. Add it to " + f"_ISOTONIC_LABEL_MAP or to _ISOTONIC_IGNORED — a fall-through " + f"silently deletes gold and inflates precision.", + ) + def _strip_index(label: str) -> str: """Isotonic labels are like `CITY_1`, `FIRSTNAME_2` — strip the trailing `_`.""" @@ -466,9 +592,9 @@ def _load_isotonic( start, end, raw_label = int(entry[0]), int(entry[1]), str(entry[2]) if raw_label == "O": continue # explicit "no label" sentinel from upstream - mapped = _ISOTONIC_LABEL_MAP.get(_strip_index(raw_label).upper()) + mapped = _map_isotonic_label(raw_label) if mapped is None: - continue # label intentionally outside our 12-class taxonomy + continue # label deliberately outside our 14-label schema spans.append(Span(mapped, start, end)) samples.append(Sample(text=text, spans=tuple(spans))) citation = "Isotonic. PII Masking 200k (open mirror of ai4privacy)." diff --git a/src/defaults.ts b/src/defaults.ts index 8aac542..cedade2 100644 --- a/src/defaults.ts +++ b/src/defaults.ts @@ -62,8 +62,9 @@ export const DEFAULT_MODEL_REVISION = 'main'; export const DEFAULT_RECOGNIZERS: readonly Recognizer[] = [ // ─── URL / Email ────────────────────────────────────────────── // URL: only http(s) + www. — bare-domain.tld dropped (FP-prone). - // The optional URL whitelist filter (PUBLIC_URL_HOSTS) lives in - // `src/url-filter.ts` and is opt-in. + // The public-host allowlist (`PUBLIC_URL_HOSTS` in `src/url-filter.ts`) + // runs as a post-filter and is on by default; opt out via + // `NullPiiConfig.urlAllowlist: 'none'`. { id: 'core:url', pattern: /\b(?:https?:\/\/|www\.)[^\s<>"]+/g, diff --git a/src/geo-filter.ts b/src/geo-filter.ts new file mode 100644 index 0000000..5ee4cca --- /dev/null +++ b/src/geo-filter.ts @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: Apache-2.0 + +import { latLonPairInRange } from './validators.js'; + +/** Shape gate for model-emitted `private_geolocation` spans. + * + * This schema defines `private_geolocation` as a coordinate — the three + * recognizers that own the label (`core:geo-latlon-decimal`, + * `core:geo-dms`, `core:geo-context` in `defaults.ts`) all match + * lat/lon literals, and `core:geo-latlon-decimal` additionally + * range-validates. The GLiNER head, which is prompted with + * `private_geolocation` zero-shot and never trained on it, does not + * share that definition: it labels place names, regions and postcodes + * as geolocation. + * + * Measured on `isotonic-en-heldout`: of 101 model-emitted + * `private_geolocation` spans, 70 were wrong — 42 sat on gold + * `private_address` (`Nidwalden`, `Emilia-Romagna`, `Rohnert Park`, + * `98025`), 11 on gold `private_ip` (`210.193.85.111`), 10 on no gold + * at all (`Southeast`, `143cm`). All 78 gold spans of the class, and + * all 31 correct predictions, are lat/lon pairs. The cost is therefore + * asymmetric: the gate removes false positives and cannot remove a true + * positive that matches the schema's own definition of the label. + * + * The damage is doubled, which is why dropping is not enough on its + * own: a place name tagged `private_geolocation` is both a false + * positive on that label AND leaves the `private_address` gold + * unmatched. Applying this filter BEFORE cross-label dedupe lets the + * model's competing `private_address` candidate for the same region + * survive instead of losing the overlap to a label it should never + * have won. + * + * Recognizer spans are not passed through this gate — they match a + * coordinate pattern by construction. + */ + +/** A decimal `lat,lon` pair, tolerant of the brackets and stray leading + * punctuation the model's span boundaries sometimes include + * (`- [-64.6681,-23.7374`). At least one component must carry a decimal + * point so bare integer pairs (`12, 34` — a date, a score, a range) + * do not qualify. */ +const DECIMAL_PAIR = + /(-?\d{1,3}\.\d+)\s*[,;]\s*(-?\d{1,3}(?:\.\d+)?)|(-?\d{1,3}(?:\.\d+)?)\s*[,;]\s*(-?\d{1,3}\.\d+)/; + +/** Degrees/minutes/seconds with a hemisphere letter — distinctive + * enough to stand alone, mirroring `core:geo-dms`. */ +const DMS = /\d{1,3}\s*°\s*\d{1,2}\s*['′]/; + +/** + * True when `text` carries a coordinate under this schema's definition + * of `private_geolocation`. + * + * @param text - the span's surface text + */ +export function isCoordinateShaped(text: string): boolean { + if (DMS.test(text)) return true; + const m = DECIMAL_PAIR.exec(text); + if (m === null) return false; + const lat = m[1] ?? m[3]; + const lon = m[2] ?? m[4]; + if (lat === undefined || lon === undefined) return false; + return latLonPairInRange(`${lat},${lon}`); +} + +/** + * Drop model-emitted `private_geolocation` spans that carry no + * coordinate. Spans of every other label pass through untouched. + * + * Call this on raw decoder output, before cross-label dedupe — see the + * module doc for why the ordering matters. + * + * @param spans - decoded model spans, each carrying its surface text + */ +export function dropNonCoordinateGeolocation< + T extends { label: string; start: number; end: number }, +>(spans: T[], text: string): T[] { + return spans.filter( + (s) => s.label !== 'private_geolocation' || isCoordinateShaped(text.slice(s.start, s.end)), + ); +} diff --git a/src/nullpii.ts b/src/nullpii.ts index 2ea9a1f..29001df 100644 --- a/src/nullpii.ts +++ b/src/nullpii.ts @@ -16,6 +16,7 @@ import { MAX_INPUT_BYTES, } from './defaults.js'; import { ModelNotInitializedError, TextTooLongError } from './errors.js'; +import { dropNonCoordinateGeolocation } from './geo-filter.js'; import { decodeGlinerLogits } from './gliner-decoder.js'; import { buildSpanCandidates } from './gliner-spans.js'; import { @@ -40,6 +41,7 @@ import { type SanitizeOptions, type SanitizeResult, } from './types/index.js'; +import { dropCleanPublicUrlSpans } from './url-filter.js'; import { PiiVault } from './vault.js'; const LOG_SCOPE = 'nullpii'; @@ -161,7 +163,14 @@ export class NullPii { }); } } - const decoded = dedupeOverlappingSpans(decodedRaw); + // Gate `private_geolocation` BEFORE any dedupe: the label is + // prompted zero-shot and the head applies it to place names and + // postcodes, which are `private_address`. Filtering here — rather + // than after — lets the model's own `private_address` candidate for + // the same region survive the overlap instead of losing it to a + // label that should never have won. See `geo-filter.ts`. + const geoFiltered = dropNonCoordinateGeolocation(decodedRaw, normalized); + const decoded = dedupeOverlappingSpans(geoFiltered); // Remap span offsets from the normalised text back to the escaped // text so they align with the regex pack and vault output. @@ -220,11 +229,24 @@ export class NullPii { // mislabels a known pattern (e.g., `ghp_…` token classified as // `account_number`) — recognizer's `secret` (0.99) overrides ML's // `account_number` (0.5–0.7). - const combined = dedupeOverlappingSpans( - [...mlSpans, ...recoSpans] as PiiSpan[], - DEFAULT_DEDUPE_IOU, - { acrossLabels: true }, - ) as PiiSpan[]; + // Drop URLs pointing at well-known public reference / documentation + // hosts (github.com, anthropic.com, docs.*, …). These leak no PII + // on their own and otherwise erode the user's token budget when a + // system prompt cites a project repo or API doc page. + // + // Runs BEFORE dedupe, and only for URLs that carry no nested PII. + // `core:url` is greedy, so `https://github.com/a,AKIA…` is a single + // span; `removeContainedSpans` inside the dedupe would delete the + // nested `secret` regardless of score, and dropping the surviving + // URL span would then emit the key as plaintext. Filtering here, + // while the nested spans still exist, keeps the check honest — see + // `dropCleanPublicUrlSpans`. + const candidates = [...mlSpans, ...recoSpans] as PiiSpan[]; + const allowlisted = + this.config.urlAllowlist === 'none' ? candidates : dropCleanPublicUrlSpans(candidates); + const combined = dedupeOverlappingSpans(allowlisted, DEFAULT_DEDUPE_IOU, { + acrossLabels: true, + }) as PiiSpan[]; const merged = applyThresholds( combined, this.config.threshold ?? DEFAULT_POST_FILTER_THRESHOLD, diff --git a/src/types/config.ts b/src/types/config.ts index 15c69f8..13be84d 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -64,4 +64,10 @@ export interface NullPiiConfig { /** Trim whitespace + common punctuation from span edges as a final * post-pass. Default: `true`. */ readonly boundaryRefine?: boolean; + /** Public-URL allowlist mode. Default: built-in {@link + * src/url-filter.PUBLIC_URL_HOSTS} drops URLs to well-known reference + * domains (github.com, anthropic.com, docs.python.org, …) from the + * `private_url` output. Pass `'none'` to disable and have every URL + * the recognizer pack and model emit treated as PII. */ + readonly urlAllowlist?: 'none'; } diff --git a/src/url-filter.ts b/src/url-filter.ts new file mode 100644 index 0000000..645c191 --- /dev/null +++ b/src/url-filter.ts @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: Apache-2.0 + +import type { PiiSpan } from './types/index.js'; + +/** + * Hosts whose URLs are public reference / documentation surfaces and + * are never PII on their own. Matching `private_url` spans are dropped + * from the final span set so that, e.g., a link to + * `https://github.com/anthropics/claude-code/issues` in a system prompt + * stays verbatim instead of becoming `{{PII_PRIVATE_URL_*}}`. + * + * Match is on the URL's host (case-insensitive). A leading `www.` is + * stripped before lookup. Subdomains of an allowlisted host also match + * (`docs.python.org` is covered by `python.org`). + */ +export const PUBLIC_URL_HOSTS: ReadonlySet = new Set([ + // Source hosting + 'github.com', + 'gitlab.com', + 'bitbucket.org', + 'codeberg.org', + 'sourceforge.net', + // Package registries + 'npmjs.com', + 'pypi.org', + 'crates.io', + 'rubygems.org', + 'packagist.org', + 'nuget.org', + 'maven.org', + // Docs / encyclopedias + 'wikipedia.org', + 'wikimedia.org', + 'mozilla.org', + 'developer.mozilla.org', + 'w3.org', + 'whatwg.org', + 'rfc-editor.org', + 'ietf.org', + // AI / ML vendors (public marketing / docs surfaces) + 'anthropic.com', + 'openai.com', + 'huggingface.co', + 'deepmind.com', + 'mistral.ai', + // Language / runtime official sites + 'python.org', + 'nodejs.org', + 'rust-lang.org', + 'golang.org', + 'go.dev', + 'oracle.com', + 'kotlinlang.org', + 'scala-lang.org', + // Cloud vendor docs (the marketing top-level — not customer subdomains) + 'aws.amazon.com', + 'cloud.google.com', + 'azure.microsoft.com', + 'docs.microsoft.com', + 'learn.microsoft.com', + // Q&A + 'stackoverflow.com', + 'stackexchange.com', + 'serverfault.com', + 'superuser.com', + // Standards bodies + 'iso.org', + 'unicode.org', +]); + +/** + * Returns `true` when `urlText` (a substring matched by the URL + * recognizer) targets one of the {@link PUBLIC_URL_HOSTS}, OR a + * subdomain of one. Returns `false` on malformed input — over-redact + * rather than under-redact. + */ +export function isPublicUrl(urlText: string): boolean { + // `core:url` is greedy (`[^\s<>"]+`) and does not stop at `,` or `#`, + // so a single span can cover several concatenated URLs: + // `https://github.com/foo,https://acme.io/internal`. `extractHost` + // only ever reads the first one, which would allowlist the whole + // blob on the strength of a host the later URLs do not share. Refuse + // to judge a span carrying more than one scheme — over-redact. + if ((urlText.match(SCHEME_PATTERN) ?? []).length > 1) return false; + const host = extractHost(urlText); + if (host === null) return false; + const lower = host.toLowerCase(); + const stripped = lower.startsWith('www.') ? lower.slice(4) : lower; + if (PUBLIC_URL_HOSTS.has(stripped)) return true; + // Subdomain match: walk parents (`docs.python.org` → `python.org`). + let cursor = stripped; + while (cursor.includes('.')) { + const dot = cursor.indexOf('.'); + cursor = cursor.slice(dot + 1); + if (PUBLIC_URL_HOSTS.has(cursor)) return true; + } + return false; +} + +/** Matches every URL scheme occurrence in a span — see {@link isPublicUrl}. */ +const SCHEME_PATTERN = /https?:\/\//gi; + +/** + * Drop `private_url` spans that target an allowlisted host **and carry + * no other PII inside them**. + * + * The containment condition is the security-critical half. `core:url` + * matches `[^\s<>"]+`, so any PII glued to a URL without whitespace is + * swallowed into the URL span — and `removeContainedSpans` then deletes + * the inner `secret` / `private_email` span regardless of its score. + * Dropping the surviving URL span would therefore emit the nested + * secret as plaintext: + * + * ``` + * https://github.com/a,AKIAIOSFODNN7EXAMPLE → AWS key in the clear + * https://user:ghp_…@github.com/org/repo.git → GitHub PAT in the clear + * https://anthropic.com/x?key=sk-ant-api03-… → Anthropic key in the clear + * ``` + * + * A 20-character allowlisted prefix would otherwise disable redaction + * for an arbitrary secret. So the rule is fail-safe: a reference URL + * that embeds anything identifying reverts to whole-URL redaction, and + * only a genuinely clean reference URL survives into the output. + * + * Must run **before** cross-label dedupe, while the nested spans still + * exist to be seen. + * + * @param spans - candidate spans, ML and recognizer, pre-dedupe + */ +export function dropCleanPublicUrlSpans(spans: readonly PiiSpan[]): PiiSpan[] { + return spans.filter((s) => { + if (s.label !== 'private_url' || !isPublicUrl(s.text)) return true; + return spans.some( + (t) => t !== s && t.label !== 'private_url' && t.start >= s.start && t.end <= s.end, + ); + }); +} + +function extractHost(urlText: string): string | null { + // The recognizer pattern allows `https?://` and `www.` prefixes. Try + // both shapes — URL constructor needs a scheme. `.hostname` not + // `.host`: the latter keeps a non-default port (`github.com:8443`), + // which misses the set lookup and then misses the parent walk too + // (`com:8443`), silently disabling the allowlist on any such URL. + try { + if (/^https?:\/\//i.test(urlText)) { + return new URL(urlText).hostname; + } + if (/^www\./i.test(urlText)) { + return new URL(`http://${urlText}`).hostname; + } + } catch { + return null; + } + return null; +} diff --git a/test/geo-filter.test.ts b/test/geo-filter.test.ts new file mode 100644 index 0000000..a01347d --- /dev/null +++ b/test/geo-filter.test.ts @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from 'vitest'; +import { dropNonCoordinateGeolocation, isCoordinateShaped } from '../src/geo-filter.js'; + +describe('isCoordinateShaped', () => { + it('accepts the decimal lat/lon pairs the model actually emits', () => { + // Verbatim span surfaces from `isotonic-en-heldout` predictions. + for (const s of ['-17.2941,-98.1523', '30.4115,-55.8172', '13.018,162.6403']) { + expect(isCoordinateShaped(s), s).toBe(true); + } + }); + + it('tolerates the bracket / stray-punctuation boundaries the head produces', () => { + expect(isCoordinateShaped('[62.4693,169.5474]')).toBe(true); + expect(isCoordinateShaped('- [-64.6681,-23.7374')).toBe(true); + }); + + it('accepts DMS with a hemisphere letter', () => { + expect(isCoordinateShaped('48° 51\' 29" N')).toBe(true); + }); + + it('rejects the place names and postcodes the zero-shot head mislabels', () => { + // Each of these was emitted as `private_geolocation` while the gold + // said `private_address`. + for (const s of ['Nidwalden', 'Emilia-Romagna', 'Rohnert Park', '98025', 'Southeast']) { + expect(isCoordinateShaped(s), s).toBe(false); + } + }); + + it('rejects IPv4 — dotted quads have no separator and must not read as a pair', () => { + expect(isCoordinateShaped('210.193.85.111')).toBe(false); + expect(isCoordinateShaped('197.54.143.140')).toBe(false); + }); + + it('rejects bare integer pairs — a range or score is not a coordinate', () => { + expect(isCoordinateShaped('12, 34')).toBe(false); + expect(isCoordinateShaped('95935500')).toBe(false); + }); + + it('defers to latLonPairInRange for range and null-island rejection', () => { + expect(isCoordinateShaped('200.5,-300.2')).toBe(false); // lat out of range + expect(isCoordinateShaped('0.0,0.0')).toBe(false); // null island / sensor default + }); +}); + +describe('dropNonCoordinateGeolocation', () => { + const text = 'Seen at 45.4642,9.1900 near Rohnert Park, IP 210.193.85.111'; + const at = (needle: string) => { + const start = text.indexOf(needle); + return { start, end: start + needle.length }; + }; + + it('keeps coordinate spans and drops the rest', () => { + const spans = [ + { label: 'private_geolocation', ...at('45.4642,9.1900') }, + { label: 'private_geolocation', ...at('Rohnert Park') }, + { label: 'private_geolocation', ...at('210.193.85.111') }, + ]; + const kept = dropNonCoordinateGeolocation(spans, text); + expect(kept.map((s) => text.slice(s.start, s.end))).toEqual(['45.4642,9.1900']); + }); + + it('never touches spans of another label', () => { + const spans = [ + { label: 'private_address', ...at('Rohnert Park') }, + { label: 'private_ip', ...at('210.193.85.111') }, + { label: 'private_geolocation', ...at('Rohnert Park') }, + ]; + const kept = dropNonCoordinateGeolocation(spans, text); + expect(kept.map((s) => s.label)).toEqual(['private_address', 'private_ip']); + }); + + it('is a no-op when no geolocation span is present', () => { + const spans = [{ label: 'private_ip', ...at('210.193.85.111') }]; + expect(dropNonCoordinateGeolocation(spans, text)).toEqual(spans); + }); +}); diff --git a/test/nullpii.test.ts b/test/nullpii.test.ts index e91252d..30961ad 100644 --- a/test/nullpii.test.ts +++ b/test/nullpii.test.ts @@ -198,6 +198,34 @@ describe('NullPii e2e pipeline (mocked ONNX)', () => { await n.dispose(); }); + it('keeps public-host URLs verbatim, redacts private hosts', async () => { + // github.com / docs.python.org are in PUBLIC_URL_HOSTS — they should + // survive the round-trip untouched. A URL on an unknown host stays + // redacted as `private_url`. + const n = new NullPii({ modelDir: '/fake', backend: 'cpu' }); + const text = + 'See https://github.com/anthropics/claude-code/issues and https://acme.io/internal'; + const out = await n.sanitize(text); + // Only the acme.io URL is redacted. + const urlSpans = out.spans.filter((s) => s.label === 'private_url'); + expect(urlSpans).toHaveLength(1); + expect(urlSpans[0]?.text).toBe('https://acme.io/internal'); + expect(out.sanitized).toContain('https://github.com/anthropics/claude-code/issues'); + expect(out.sanitized).not.toContain('https://acme.io/internal'); + const restored = n.restore(out.sanitized, out.sessionId); + expect(restored.restored).toBe(text); + await n.dispose(); + }); + + it('urlAllowlist: "none" redacts every URL including public hosts', async () => { + const n = new NullPii({ modelDir: '/fake', backend: 'cpu', urlAllowlist: 'none' }); + const text = 'See https://github.com/foo and https://acme.io/internal'; + const out = await n.sanitize(text); + const urlSpans = out.spans.filter((s) => s.label === 'private_url'); + expect(urlSpans).toHaveLength(2); + await n.dispose(); + }); + it('init runs once across many sanitize calls', async () => { const n = new NullPii({ modelDir: '/fake', backend: 'cpu' }); const r1 = await n.sanitize('First email: a@acme.io'); diff --git a/test/url-filter-nested.test.ts b/test/url-filter-nested.test.ts new file mode 100644 index 0000000..6c224f6 --- /dev/null +++ b/test/url-filter-nested.test.ts @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from 'vitest'; +import type { PiiSpan } from '../src/types/index.js'; +import { dropCleanPublicUrlSpans, isPublicUrl } from '../src/url-filter.js'; + +/** `core:url` is greedy (`[^\s<>"]+`), so PII glued to a URL without + * whitespace lands inside the URL span. Dropping such a span emits the + * nested secret as plaintext — verified end-to-end against the real + * model before this guard existed: + * + * x https://github.com/a,AKIAIOSFODNN7EXAMPLE y + * main → x {{PII_PRIVATE_URL_0}} y + * allowlist, pre → x https://github.com/a,AKIAIOSFODNN7EXAMPLE y ← leak + * allowlist, post → x {{PII_PRIVATE_URL_0}} y + */ +const span = (label: string, start: number, end: number, text: string): PiiSpan => + ({ label, start, end, text, score: 0.95 }) as PiiSpan; + +describe('dropCleanPublicUrlSpans', () => { + it('drops a clean allowlisted URL — the feature this exists for', () => { + const url = span('private_url', 5, 41, 'https://github.com/acme/infra-gitops'); + expect(dropCleanPublicUrlSpans([url])).toEqual([]); + }); + + it('keeps an allowlisted URL that carries a nested secret', () => { + const url = span('private_url', 2, 43, 'https://github.com/a,AKIAIOSFODNN7EXAMPLE'); + const key = span('secret', 23, 43, 'AKIAIOSFODNN7EXAMPLE'); + // Both survive the filter; cross-label dedupe then collapses them to + // the outer URL span, so the whole URL is redacted rather than the + // key being emitted in the clear. + expect(dropCleanPublicUrlSpans([url, key])).toHaveLength(2); + }); + + it('keeps an allowlisted URL carrying nested credentials in userinfo', () => { + const text = 'https://user:ghp_ABCDEF@github.com/org/repo.git'; + const url = span('private_url', 0, text.length, text); + const pat = span('secret', 13, 23, 'ghp_ABCDEF'); + expect(dropCleanPublicUrlSpans([url, pat])).toHaveLength(2); + }); + + it('never touches spans of another label', () => { + const email = span('private_email', 0, 13, 'a@example.com'); + expect(dropCleanPublicUrlSpans([email])).toEqual([email]); + }); + + it('keeps a non-allowlisted URL even when clean', () => { + const url = span('private_url', 0, 29, 'https://intranet.acme.local/x'); + expect(dropCleanPublicUrlSpans([url])).toEqual([url]); + }); + + it('ignores a nested span of the same label — only foreign PII counts', () => { + const url = span('private_url', 0, 36, 'https://github.com/acme/infra-gitops'); + const inner = span('private_url', 8, 18, 'github.com'); + expect(dropCleanPublicUrlSpans([url, inner])).toEqual([inner]); + }); +}); + +describe('isPublicUrl — concatenated URLs', () => { + it('refuses to allowlist a span carrying more than one scheme', () => { + // One greedy span covers both; judging it by the first host would + // allowlist the internal one on github.com's authority. + expect(isPublicUrl('https://github.com/foo,https://acme.io/internal')).toBe(false); + expect(isPublicUrl('https://github.com/x?next=https://acme.io/internal')).toBe(false); + }); + + it('still allowlists a single URL', () => { + expect(isPublicUrl('https://github.com/acme/infra-gitops')).toBe(true); + }); +}); + +describe('isPublicUrl — host extraction', () => { + it('allowlists a non-default port (hostname, not host)', () => { + expect(isPublicUrl('https://github.com:8443/foo')).toBe(true); + }); + + it('still rejects the near-miss hosts', () => { + for (const u of [ + 'https://evil-github.com/x', + 'https://github.com.attacker.net/x', + 'https://acme-corp.github.io/private', + ]) { + expect(isPublicUrl(u), u).toBe(false); + } + }); +}); diff --git a/test/url-filter.test.ts b/test/url-filter.test.ts new file mode 100644 index 0000000..0708daf --- /dev/null +++ b/test/url-filter.test.ts @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from 'vitest'; +import type { PiiSpan } from '../src/types/index.js'; +import { dropCleanPublicUrlSpans, isPublicUrl } from '../src/url-filter.js'; + +/** Spans must carry realistic, non-overlapping offsets: the drop rule is + * containment-aware, so a fixture that parks every span at 0 would read + * as "this URL has PII nested inside it" and keep everything. */ +function urlSpan(text: string, start = 0): PiiSpan { + return { label: 'private_url', start, end: start + text.length, text, score: 0.95 }; +} + +describe('isPublicUrl', () => { + it('matches direct hosts (github.com, anthropic.com)', () => { + expect(isPublicUrl('https://github.com/foo/bar')).toBe(true); + expect(isPublicUrl('https://anthropic.com/news')).toBe(true); + }); + + it('matches subdomains of allowlisted hosts', () => { + expect(isPublicUrl('https://docs.python.org/3/library/')).toBe(true); + expect(isPublicUrl('https://developer.mozilla.org/en-US/')).toBe(true); + }); + + it('strips `www.` prefix before lookup', () => { + expect(isPublicUrl('https://www.wikipedia.org/wiki/Foo')).toBe(true); + expect(isPublicUrl('www.github.com/issues')).toBe(true); + }); + + it('rejects unrelated hosts', () => { + expect(isPublicUrl('https://acme.io/internal/dashboard')).toBe(false); + expect(isPublicUrl('https://example.com/foo')).toBe(false); + }); + + it('rejects malformed input (over-redact stance)', () => { + expect(isPublicUrl('not a url')).toBe(false); + expect(isPublicUrl('javascript:alert(1)')).toBe(false); + expect(isPublicUrl('')).toBe(false); + }); +}); + +describe('dropCleanPublicUrlSpans', () => { + it('drops private_url spans matching the allowlist, keeps others', () => { + const spans: PiiSpan[] = [ + urlSpan('https://github.com/foo', 0), + urlSpan('https://acme.io/internal', 30), + { label: 'private_email', start: 60, end: 67, text: 'a@b.com', score: 0.95 }, + ]; + const out = dropCleanPublicUrlSpans(spans); + expect(out).toHaveLength(2); + expect(out.find((s) => s.text === 'https://github.com/foo')).toBeUndefined(); + expect(out.find((s) => s.text === 'https://acme.io/internal')).toBeDefined(); + expect(out.find((s) => s.label === 'private_email')).toBeDefined(); + }); + + it('passes through when no private_url spans are present', () => { + const spans: PiiSpan[] = [ + { label: 'private_email', start: 0, end: 7, text: 'a@b.com', score: 0.95 }, + ]; + expect(dropCleanPublicUrlSpans(spans)).toEqual(spans); + }); +});