From c3e6daf14e3e4203699442a453fb31d95f425ebc Mon Sep 17 00:00:00 2001 From: Richard Hope Date: Fri, 31 Jul 2026 17:50:52 +1000 Subject: [PATCH] api: configurable data retention for findings and audit events (#73) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Findings, their append-only trail, and the administrative audit log grow forever. This bounds that — and, mostly, defines what will never be deleted. **Every window is off by default.** This database is evidence: it records where an organisation's secrets are and who did what about each one. Deleting some of it has to be a decision somebody made, not something that starts happening because they upgraded. With nothing configured the purge runs and removes nothing. **Never eligible, whatever the window says:** * open findings — an unresolved secret is not old news; * analyst decisions (`false_positive`, `accepted_risk`, *manual* resolutions) — a judgement that expires silently means the finding returns next scan with nobody remembering it was already considered. Only `auto` resolutions, which are an inference from absence, are eligible; * suppressed findings — deleting one resurrects it as new on the next scan, the exact outcome the suppression exists to prevent (ADR 0008); * an open finding's events, however old. The finding clock runs from the decision, not the first sighting: a secret found two years ago and auto-resolved yesterday is one day old here. Purges audit themselves. Deleting evidence is an administrative action, so a round that removed anything writes `retention.purged` with the counts *and the windows that justified them*, so the row still explains itself when the settings have since changed. A round that deleted nothing writes nothing — an audit log full of "deleted 0 rows" every minute is one nobody reads. That row is subject to the audit window like any other, which is honest rather than a special case hiding from the policy it enforces. Runs last in the maintenance round, under the existing advisory lock, and deletes in batches so the first purge on a database that has never had one cannot hold locks for minutes. `python -m iceberg_api retention-purge` runs one now. `docs/retention.md` covers the windows, the never-delete rules, and guidance on choosing them (including that "you cannot get these rows back"). Closes #73 Co-Authored-By: Claude Opus 5 (1M context) --- .env.example | 13 + apps/api/src/iceberg_api/cli.py | 17 +- apps/api/src/iceberg_api/maintenance.py | 13 +- apps/api/src/iceberg_api/retention.py | 202 ++++++++++ apps/api/tests/test_maintenance.py | 12 +- apps/api/tests/test_retention.py | 367 ++++++++++++++++++ deploy/compose/docker-compose.yml | 5 + docs/retention.md | 93 +++++ packages/core/src/iceberg_core/config.py | 22 ++ .../core/src/iceberg_core/models/__init__.py | 4 + .../core/src/iceberg_core/models/audit.py | 5 + 11 files changed, 745 insertions(+), 8 deletions(-) create mode 100644 apps/api/src/iceberg_api/retention.py create mode 100644 apps/api/tests/test_retention.py create mode 100644 docs/retention.md diff --git a/.env.example b/.env.example index dc402ad..6fe9ff2 100644 --- a/.env.example +++ b/.env.example @@ -103,6 +103,19 @@ ICEBERG_ENGINE_AUTH_RATE_LIMIT=30 # THIS MUST BE SET, or every request is charged to the balancer's address. ICEBERG_TRUSTED_PROXY_HOPS=0 +# ─── Data retention (api role, #73) ─────────────────────────────────────────── +# All three windows are OFF by default, and that is deliberate: this database +# records where your secrets are and who did what about them, so deleting any of +# it should be a decision, not an upgrade side effect (docs/retention.md). +# +# 0 = keep forever. Open findings, analyst decisions (false_positive, +# accepted_risk, manual resolutions) and suppressed findings are never eligible +# whatever these say. +ICEBERG_RETENTION_RESOLVED_FINDINGS_DAYS=0 +ICEBERG_RETENTION_FINDING_EVENTS_DAYS=0 +# Usually the window a compliance regime names. You cannot get these rows back. +ICEBERG_RETENTION_AUDIT_EVENTS_DAYS=0 + # ─── Engine role ────────────────────────────────────────────────────────────── # The per-engine API credential. In production this is minted at deploy time # (docs/security.md § Bootstrap), never defaulted into an image. diff --git a/apps/api/src/iceberg_api/cli.py b/apps/api/src/iceberg_api/cli.py index 4495e87..84f1d8a 100644 --- a/apps/api/src/iceberg_api/cli.py +++ b/apps/api/src/iceberg_api/cli.py @@ -32,7 +32,7 @@ ) from sqlmodel import col, select -from iceberg_api import audit +from iceberg_api import audit, retention from iceberg_api.dispatch import build_dispatcher from iceberg_api.engines.auth import mint_token from iceberg_api.scans import service @@ -58,6 +58,10 @@ def _build_parser() -> argparse.ArgumentParser: commands.add_parser("reclaim", help="return expired-lease tasks to the queue") commands.add_parser("scheduler-tick", help="run one scheduler round now") + commands.add_parser( + "retention-purge", + help="apply the configured retention windows now (see docs/retention.md)", + ) migrate_parser = commands.add_parser("migrate", help="apply migrations up to a revision") migrate_parser.add_argument( @@ -161,6 +165,17 @@ def main(argv: Sequence[str] | None = None) -> int: case "migrate": migrate(args.revision) print(f"migrated to {args.revision}", file=sys.stderr) + case "retention-purge": + # The maintenance loop runs this on its own cadence; this exists for + # the first purge after a window is configured, when an operator wants + # to see the number rather than discover it in the audit log. + with session_scope() as db: + purged = retention.purge(db, settings) + print( + f"findings={purged.findings} finding_events={purged.finding_events} " + f"audit_events={purged.audit_events}", + file=sys.stderr, + ) return 0 diff --git a/apps/api/src/iceberg_api/maintenance.py b/apps/api/src/iceberg_api/maintenance.py index 0005519..ee9166a 100644 --- a/apps/api/src/iceberg_api/maintenance.py +++ b/apps/api/src/iceberg_api/maintenance.py @@ -26,7 +26,7 @@ from iceberg_core.db import session_scope from iceberg_core.secrets import SecretStore, build_secret_store -from iceberg_api import suppressions +from iceberg_api import retention, suppressions from iceberg_api.dispatch import Dispatcher, build_dispatcher from iceberg_api.notifications import dispatch as notification_dispatch from iceberg_api.scans import service @@ -43,7 +43,7 @@ def run_once( settings: ApiSettings | None = None, store: SecretStore | None = None, ) -> None: - """One maintenance round: schedules, reclaim, the safety sweeps, then alerts. + """One round: schedules, reclaim, the safety sweeps, alerts, then retention. Leadership is held on a session of its own for the whole round. Holding it on the working session would not work: ``pg_try_advisory_xact_lock`` releases at @@ -53,7 +53,8 @@ def run_once( transaction — and the lock — spans everything below. ``settings``/``store`` are injectable so a test can drive a round without a - configured SMTP relay; both default to the process configuration. + configured SMTP relay and with its own retention windows; both default to the + process configuration. """ at = now or datetime.now(UTC) resolved = settings or get_api_settings() @@ -88,6 +89,12 @@ def run_once( # gets retried on the next beat instead of losing the alert. with session_scope() as db: notification_dispatch.deliver_pending(db, resolved, secret_store, now=at) + # Retention (#73). A no-op unless the deployment configured a window — + # this database is evidence, so deleting any of it is opt-in. Last in the + # round because it is the only job that can be slow on a database that has + # never been purged, and nothing else waits on it. + with session_scope() as db: + retention.purge(db, resolved, now=at) def _already_leader(db: object) -> bool: diff --git a/apps/api/src/iceberg_api/retention.py b/apps/api/src/iceberg_api/retention.py new file mode 100644 index 0000000..b1c8f86 --- /dev/null +++ b/apps/api/src/iceberg_api/retention.py @@ -0,0 +1,202 @@ +"""Data retention (#73). + +Findings and their append-only event trail grow forever, and so does the +administrative audit log. This deletes what a deployment has decided it no longer +needs — and nothing else, which is most of the design. + +**Every window is off by default.** This database is evidence: it records where +secrets were found and who did what about them. Deleting some of it has to be a +decision somebody made, not something that starts happening because they +upgraded. With no windows configured this module does nothing at all. + +**What is never eligible**, whatever the window says: + +* **Open findings.** An unresolved secret is not old news, it is a secret nobody + has dealt with. +* **Findings an analyst decided about** — ``false_positive``, ``accepted_risk``, + or a manual resolution. Those are judgements, and a judgement that expires + silently means the same finding comes back next scan with nobody remembering + it was already considered. Only ``auto`` resolutions — an inference from + absence, made by a scan — are eligible. +* **Events belonging to a finding that still exists and is open.** Its trail is + how an analyst understands the state they are looking at. +* **A suppressed finding's row**, because the suppression is the reason it is not + in anyone's list; deleting it would resurrect it on the next scan as new. + +**The purge is itself audited.** Deleting evidence is an administrative action, +so each round that removes anything writes an ``audit_event`` recording what was +deleted, how much, and the window that justified it — and that row is subject to +the audit window like any other, which is the honest behaviour rather than a +special case that hides purges from the retention it enforces. + +Deletion is batched (``retention_batch_size``) so the first purge on a database +that has never had one cannot hold locks for minutes. A round that hits the batch +ceiling simply continues on the next beat. +""" + +from dataclasses import asdict, dataclass +from datetime import UTC, datetime, timedelta + +import structlog +from iceberg_core.config import ApiSettings +from iceberg_core.enums import FindingResolution, FindingState +from iceberg_core.models import ( + AUDIT_RETENTION_PURGED, + AUDIT_TARGET_RETENTION, + AuditEvent, + Finding, + FindingEvent, +) +from sqlmodel import Session, col, delete, select + +from iceberg_api import audit + +logger = structlog.get_logger() + +#: Resolutions a purge may remove. Deliberately only this one — see the module +#: docstring. A tuple rather than a set so the membership test reads in the query. +PURGEABLE_RESOLUTIONS = (FindingResolution.AUTO,) + + +@dataclass(frozen=True, slots=True) +class PurgeResult: + """What one retention round deleted.""" + + findings: int = 0 + finding_events: int = 0 + audit_events: int = 0 + + def total(self) -> int: + return self.findings + self.finding_events + self.audit_events + + +def purge( + db: Session, + settings: ApiSettings, + *, + now: datetime | None = None, +) -> PurgeResult: + """Delete what has aged out. Commits, and audits itself if it removed anything.""" + at = now or datetime.now(UTC) + batch = settings.retention_batch_size + + findings = _purge_findings(db, settings.retention_resolved_findings_days, at=at, batch=batch) + # After the findings, so events cascaded away with their finding are not + # counted twice — and so the count reported here is events pruned *from + # surviving findings*, which is the number an operator is asking about. + events = _purge_finding_events(db, settings.retention_finding_events_days, at=at, batch=batch) + audits = _purge_audit_events(db, settings.retention_audit_events_days, at=at, batch=batch) + + result = PurgeResult(findings=findings, finding_events=events, audit_events=audits) + if result.total() == 0: + db.commit() + return result + + audit.record( + db, + # No actor: nobody triggered this, a policy did. + actor_id=None, + action=AUDIT_RETENTION_PURGED, + target_type=AUDIT_TARGET_RETENTION, + target_id=None, + detail={ + **{key: str(value) for key, value in asdict(result).items()}, + # The window that justified it, so the row still explains itself when + # somebody reads it a year later and the settings have changed. + "resolved_findings_days": str(settings.retention_resolved_findings_days), + "finding_events_days": str(settings.retention_finding_events_days), + "audit_events_days": str(settings.retention_audit_events_days), + }, + ) + db.commit() + logger.info("retention_purged", **asdict(result)) + return result + + +def _cutoff(days: int, at: datetime) -> datetime | None: + """The instant before which rows are eligible, or None when disabled.""" + return None if days <= 0 else at - timedelta(days=days) + + +def _purge_findings(db: Session, days: int, *, at: datetime, batch: int) -> int: + """Auto-resolved findings that have been resolved longer than the window. + + Eligibility is checked in the query rather than in Python: a finding that + changes state between a read and a delete would otherwise be removed on the + strength of a state it no longer has. + """ + cutoff = _cutoff(days, at) + if cutoff is None: + return 0 + + eligible = list( + db.exec( + select(col(Finding.id)) + .where( + col(Finding.state) == FindingState.RESOLVED, + col(Finding.resolution).in_(PURGEABLE_RESOLUTIONS), + # `updated_at` moves when the finding is resolved, so it is the + # age of the *decision*, not of the first sighting. A finding + # found two years ago and resolved yesterday is a day old here. + col(Finding.updated_at) < cutoff, + # A suppressed finding is hidden, not gone: deleting it would + # bring it back as new on the next scan (ADR 0008). + col(Finding.suppressed_at).is_(None), + ) + .limit(batch) + ) + ) + if not eligible: + return 0 + + # FindingEvent cascades on finding_id, so the trail goes with the finding it + # describes rather than being orphaned. + db.exec(delete(Finding).where(col(Finding.id).in_(eligible))) + return len(eligible) + + +def _purge_finding_events(db: Session, days: int, *, at: datetime, batch: int) -> int: + """Old trail entries for findings that are no longer open. + + An open finding keeps its whole history however old: it is what an analyst + reads to understand the state in front of them. + """ + cutoff = _cutoff(days, at) + if cutoff is None: + return 0 + + still_open = select(col(Finding.id)).where(col(Finding.state) == FindingState.OPEN) + eligible = list( + db.exec( + select(col(FindingEvent.id)) + .where( + col(FindingEvent.created_at) < cutoff, + col(FindingEvent.finding_id).not_in(still_open), + ) + .limit(batch) + ) + ) + if not eligible: + return 0 + + db.exec(delete(FindingEvent).where(col(FindingEvent.id).in_(eligible))) + return len(eligible) + + +def _purge_audit_events(db: Session, days: int, *, at: datetime, batch: int) -> int: + """Administrative audit rows past their window.""" + cutoff = _cutoff(days, at) + if cutoff is None: + return 0 + + eligible = list( + db.exec(select(col(AuditEvent.id)).where(col(AuditEvent.created_at) < cutoff).limit(batch)) + ) + if not eligible: + return 0 + + db.exec(delete(AuditEvent).where(col(AuditEvent.id).in_(eligible))) + return len(eligible) + + +__all__ = ["PURGEABLE_RESOLUTIONS", "PurgeResult", "purge"] diff --git a/apps/api/tests/test_maintenance.py b/apps/api/tests/test_maintenance.py index 9d95ddd..8cffa3b 100644 --- a/apps/api/tests/test_maintenance.py +++ b/apps/api/tests/test_maintenance.py @@ -43,9 +43,11 @@ def process_engine_fixture(db_engine: SAEngine) -> Iterator[None]: def run_round_fixture(secret_store: SecretStore) -> Callable[..., None]: """One maintenance round with settings supplied rather than read from the env. - A round delivers queued notifications (#60), so it needs settings and a secret - store. Injecting them keeps these tests from depending on a configured - deployment — and from needing an SMTP relay to prove the scheduler fires. + A round delivers queued notifications (#60) and applies retention (#73), so + it needs settings and a secret store. Injecting them keeps these tests from + depending on a configured deployment — no SMTP relay is needed to prove the + scheduler fires, and the retention windows default to off, so none of these + rounds can delete anything. """ settings = ApiSettings( database_url="postgresql+psycopg://unused/unused", @@ -94,7 +96,9 @@ def test_a_round_launches_due_scans_and_advances_the_schedule( def test_a_round_reclaims_expired_leases( - session: Session, dispatcher: RecordingDispatcher, run_round: Callable[..., None] + session: Session, + dispatcher: RecordingDispatcher, + run_round: Callable[..., None], ) -> None: source = _source(session) scan = service.launch_scan(session, source, trigger=ScanTrigger.MANUAL, dispatcher=dispatcher) diff --git a/apps/api/tests/test_retention.py b/apps/api/tests/test_retention.py new file mode 100644 index 0000000..42c8c93 --- /dev/null +++ b/apps/api/tests/test_retention.py @@ -0,0 +1,367 @@ +"""Data retention (#73). + +Almost every test here asserts that something was **not** deleted. That is the +right shape for this feature: the purge is trivial and the eligibility rules are +the whole product, and the failure mode of getting them wrong is destroying +evidence nobody can get back. + +The rules, and why each exists: + +* **Off by default.** No window configured, nothing deleted, ever. +* **Open findings are never eligible.** An unresolved secret is not old news. +* **Only `auto` resolutions.** `false_positive` and `accepted_risk` are analyst + judgements; letting them expire means the finding returns next scan with nobody + remembering it was already considered. +* **Suppressed findings are never eligible**, because deleting one resurrects it + as new on the next scan (ADR 0008). +* **An open finding keeps its whole trail**, however old the events are. +* **The purge audits itself**, because deleting evidence is an administrative + action. +""" + +import uuid +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from typing import Any + +import pytest +from iceberg_api import retention +from iceberg_core.config import ApiSettings +from iceberg_core.enums import ( + FindingEventKind, + FindingResolution, + FindingState, + ScanStatus, + ScanTrigger, + Severity, + SourceType, +) +from iceberg_core.models import ( + AUDIT_RETENTION_PURGED, + AuditEvent, + Finding, + FindingEvent, + Scan, + Source, +) +from pydantic import SecretStr +from sqlmodel import Session, col, select, update + +NOW = datetime(2026, 7, 31, 12, 0, tzinfo=UTC) +LONG_AGO = NOW - timedelta(days=400) +RECENTLY = NOW - timedelta(days=2) + + +def _settings(**windows: int) -> ApiSettings: + return ApiSettings( + database_url="postgresql+psycopg://unused/unused", + master_key=SecretStr("unused"), + **windows, # type: ignore[arg-type] + ) + + +@pytest.fixture(name="source") +def source_fixture(session: Session) -> Source: + source = Source( + name=f"confluence-{uuid.uuid4().hex[:6]}", + type=SourceType.CONFLUENCE, + connection={"base_url": "https://example.atlassian.net/wiki"}, + ) + session.add(source) + session.commit() + return source + + +@pytest.fixture(name="scan") +def scan_fixture(session: Session, source: Source) -> Scan: + scan = Scan(source_id=source.id, trigger=ScanTrigger.MANUAL, status=ScanStatus.COMPLETED) + session.add(scan) + session.commit() + return scan + + +@pytest.fixture(name="make_finding") +def make_finding_fixture(session: Session, source: Source, scan: Scan) -> Callable[..., Finding]: + def factory(*, updated_at: datetime = LONG_AGO, **fields: Any) -> Finding: + defaults: dict[str, Any] = { + "fingerprint": uuid.uuid4().hex, + "rule_id": "aws-access-key", + "rulepack_version": "2026.07.1", + "resource_locator": {"path": "/space/DOCS/page-1"}, + "redacted_snippet": "AKIA****************", + "secret_hash": uuid.uuid4().hex, + "severity": Severity.HIGH, + "state": FindingState.RESOLVED, + "resolution": FindingResolution.AUTO, + } + finding = Finding( + source_id=source.id, + first_seen_scan_id=scan.id, + last_seen_scan_id=scan.id, + **(defaults | fields), + ) + session.add(finding) + session.commit() + # `updated_at` is maintained by the model on write, so the age is set + # afterwards with an UPDATE — the same row real time would have produced. + session.exec( + update(Finding).where(col(Finding.id) == finding.id).values(updated_at=updated_at) + ) + session.commit() + session.refresh(finding) + return finding + + return factory + + +def _age_event(session: Session, event: FindingEvent, created_at: datetime) -> None: + session.exec( + update(FindingEvent).where(col(FindingEvent.id) == event.id).values(created_at=created_at) + ) + session.commit() + + +# ── Off by default ──────────────────────────────────────────────────────────── + + +def test_nothing_is_deleted_without_a_configured_window( + session: Session, make_finding: Callable[..., Finding] +) -> None: + """The default is keep-everything, and it has to be: this database records + where an organisation's secrets are, and losing that on upgrade would be a + surprise nobody can undo.""" + make_finding() + + result = retention.purge(session, _settings(), now=NOW) + + assert result.total() == 0 + assert len(list(session.exec(select(Finding)))) == 1 + + +# ── Findings ────────────────────────────────────────────────────────────────── + + +def test_an_old_auto_resolved_finding_is_purged( + session: Session, make_finding: Callable[..., Finding] +) -> None: + make_finding() + + result = retention.purge(session, _settings(retention_resolved_findings_days=90), now=NOW) + + assert result.findings == 1 + assert list(session.exec(select(Finding))) == [] + + +def test_an_open_finding_is_never_purged( + session: Session, make_finding: Callable[..., Finding] +) -> None: + """An unresolved secret is not old news; it is a secret nobody dealt with.""" + make_finding(state=FindingState.OPEN, resolution=None) + + result = retention.purge(session, _settings(retention_resolved_findings_days=1), now=NOW) + + assert result.findings == 0 + assert len(list(session.exec(select(Finding)))) == 1 + + +@pytest.mark.parametrize( + ("state", "resolution"), + [ + (FindingState.ACCEPTED_RISK, None), + (FindingState.FALSE_POSITIVE, None), + (FindingState.RESOLVED, FindingResolution.MANUAL), + ], +) +def test_an_analyst_decision_is_never_purged( + session: Session, + make_finding: Callable[..., Finding], + state: FindingState, + resolution: FindingResolution | None, +) -> None: + """A judgement that expires silently means the finding comes back next scan + with nobody remembering it was already considered.""" + make_finding(state=state, resolution=resolution) + + result = retention.purge(session, _settings(retention_resolved_findings_days=1), now=NOW) + + assert result.findings == 0 + assert len(list(session.exec(select(Finding)))) == 1 + + +def test_a_suppressed_finding_is_never_purged( + session: Session, make_finding: Callable[..., Finding] +) -> None: + """Deleting it would resurrect it as new on the next scan, which is the exact + outcome the suppression exists to prevent (ADR 0008).""" + make_finding(suppressed_at=LONG_AGO) + + result = retention.purge(session, _settings(retention_resolved_findings_days=1), now=NOW) + + assert result.findings == 0 + + +def test_a_recently_resolved_finding_is_inside_the_window( + session: Session, make_finding: Callable[..., Finding] +) -> None: + """The clock runs from the decision, not the first sighting: a secret found + two years ago and resolved yesterday is a day old for retention.""" + make_finding(updated_at=RECENTLY) + + result = retention.purge(session, _settings(retention_resolved_findings_days=90), now=NOW) + + assert result.findings == 0 + + +def test_a_purged_finding_takes_its_trail_with_it( + session: Session, make_finding: Callable[..., Finding] +) -> None: + finding = make_finding() + session.add( + FindingEvent( + finding_id=finding.id, + kind=FindingEventKind.STATE_CHANGE, + from_value="open", + to_value="resolved", + ) + ) + session.commit() + + retention.purge(session, _settings(retention_resolved_findings_days=90), now=NOW) + + assert list(session.exec(select(FindingEvent))) == [] + + +# ── Finding events ──────────────────────────────────────────────────────────── + + +def test_an_open_findings_events_are_kept_however_old( + session: Session, make_finding: Callable[..., Finding] +) -> None: + """The trail is what an analyst reads to understand the state in front of + them, and an old open finding is exactly the one that needs explaining.""" + finding = make_finding(state=FindingState.OPEN, resolution=None) + event = FindingEvent( + finding_id=finding.id, + kind=FindingEventKind.COMMENT, + comment="asked the space owner to rotate this", + ) + session.add(event) + session.commit() + _age_event(session, event, LONG_AGO) + + result = retention.purge(session, _settings(retention_finding_events_days=30), now=NOW) + + assert result.finding_events == 0 + assert len(list(session.exec(select(FindingEvent)))) == 1 + + +def test_old_events_on_a_settled_finding_are_pruned( + session: Session, make_finding: Callable[..., Finding] +) -> None: + finding = make_finding(state=FindingState.ACCEPTED_RISK, resolution=None) + event = FindingEvent( + finding_id=finding.id, + kind=FindingEventKind.STATE_CHANGE, + from_value="open", + to_value="accepted_risk", + ) + session.add(event) + session.commit() + _age_event(session, event, LONG_AGO) + + result = retention.purge(session, _settings(retention_finding_events_days=30), now=NOW) + + assert result.finding_events == 1 + # The finding itself survives — it is an analyst decision. + assert len(list(session.exec(select(Finding)))) == 1 + + +def test_a_recent_event_is_kept(session: Session, make_finding: Callable[..., Finding]) -> None: + finding = make_finding(state=FindingState.ACCEPTED_RISK, resolution=None) + event = FindingEvent(finding_id=finding.id, kind=FindingEventKind.COMMENT, comment="recent") + session.add(event) + session.commit() + _age_event(session, event, RECENTLY) + + result = retention.purge(session, _settings(retention_finding_events_days=30), now=NOW) + + assert result.finding_events == 0 + + +# ── Audit events ────────────────────────────────────────────────────────────── + + +def test_old_audit_events_are_pruned_and_recent_ones_kept(session: Session) -> None: + old = AuditEvent(action="user.role_changed", target_type="user", target_id=uuid.uuid4()) + recent = AuditEvent(action="source.created", target_type="source", target_id=uuid.uuid4()) + session.add(old) + session.add(recent) + session.commit() + for event, when in ((old, LONG_AGO), (recent, RECENTLY)): + session.exec( + update(AuditEvent).where(col(AuditEvent.id) == event.id).values(created_at=when) + ) + session.commit() + + result = retention.purge(session, _settings(retention_audit_events_days=365), now=NOW) + + assert result.audit_events == 1 + surviving = list( + session.exec(select(AuditEvent).where(col(AuditEvent.action) != "retention.purged")) + ) + assert [event.action for event in surviving] == ["source.created"] + + +# ── The purge audits itself ─────────────────────────────────────────────────── + + +def test_a_purge_that_deleted_something_is_audited( + session: Session, make_finding: Callable[..., Finding] +) -> None: + """Deleting evidence is an administrative action. The row records what went + and the window that justified it, so it still explains itself a year later + when the settings have changed.""" + make_finding() + + retention.purge(session, _settings(retention_resolved_findings_days=90), now=NOW) + + recorded = session.exec( + select(AuditEvent).where(col(AuditEvent.action) == AUDIT_RETENTION_PURGED) + ).one() + assert recorded.actor_id is None # a policy did this, not a person + assert recorded.detail["findings"] == "1" + assert recorded.detail["resolved_findings_days"] == "90" + + +def test_a_purge_that_deleted_nothing_writes_no_audit_row(session: Session) -> None: + """An audit log full of "deleted 0 rows" every minute is one nobody reads.""" + retention.purge(session, _settings(retention_resolved_findings_days=90), now=NOW) + + assert list(session.exec(select(AuditEvent))) == [] + + +# ── Batching ────────────────────────────────────────────────────────────────── + + +def test_a_round_deletes_at_most_the_batch_size( + session: Session, make_finding: Callable[..., Finding] +) -> None: + """The first purge on a database that has never had one must not hold locks + for minutes; what it does not take this round, it takes on the next beat.""" + for _ in range(5): + make_finding() + + first = retention.purge( + session, + _settings(retention_resolved_findings_days=90, retention_batch_size=2), + now=NOW, + ) + second = retention.purge( + session, + _settings(retention_resolved_findings_days=90, retention_batch_size=2), + now=NOW, + ) + + assert first.findings == 2 + assert second.findings == 2 + assert len(list(session.exec(select(Finding)))) == 1 diff --git a/deploy/compose/docker-compose.yml b/deploy/compose/docker-compose.yml index 84a06b0..7c4264a 100644 --- a/deploy/compose/docker-compose.yml +++ b/deploy/compose/docker-compose.yml @@ -104,6 +104,11 @@ services: # 0 for the compose stack: nothing proxies the api here, so the peer # address is the client. Behind an ingress this must match the hop count. ICEBERG_TRUSTED_PROXY_HOPS: ${ICEBERG_TRUSTED_PROXY_HOPS:-0} + # Retention (#73). Off by default — deleting evidence is opt-in, and the + # api role is the only one that touches the database anyway. + ICEBERG_RETENTION_RESOLVED_FINDINGS_DAYS: ${ICEBERG_RETENTION_RESOLVED_FINDINGS_DAYS:-0} + ICEBERG_RETENTION_FINDING_EVENTS_DAYS: ${ICEBERG_RETENTION_FINDING_EVENTS_DAYS:-0} + ICEBERG_RETENTION_AUDIT_EVENTS_DAYS: ${ICEBERG_RETENTION_AUDIT_EVENTS_DAYS:-0} ports: - "${ICEBERG_API_PORT:-8000}:8000" depends_on: diff --git a/docs/retention.md b/docs/retention.md new file mode 100644 index 0000000..a80a717 --- /dev/null +++ b/docs/retention.md @@ -0,0 +1,93 @@ +# Data retention + +Findings, their append-only event trail, and the administrative audit log all grow forever. This +is how a deployment bounds that — and, more importantly, what it will never delete. + +**Every window is off by default.** This database is evidence: it records where an organisation's +secrets were found, and who did what about each one. Deleting some of it has to be a decision +somebody made, not something that starts happening because they upgraded. With nothing configured, +the purge runs and deletes nothing. + +## What can be deleted + +| Setting | Default | What ages out | +|---|---|---| +| `ICEBERG_RETENTION_RESOLVED_FINDINGS_DAYS` | `0` (keep forever) | Findings **auto-resolved** longer ago than this. | +| `ICEBERG_RETENTION_FINDING_EVENTS_DAYS` | `0` (keep forever) | `FindingEvent` rows older than this, on findings that are not open. | +| `ICEBERG_RETENTION_AUDIT_EVENTS_DAYS` | `0` (keep forever) | `AuditEvent` rows older than this. | +| `ICEBERG_RETENTION_BATCH_SIZE` | `1000` | Rows per table per round. | + +The finding clock runs from the **decision**, not the first sighting: `updated_at` moves when the +finding is resolved, so a secret found two years ago and auto-resolved yesterday is one day old +for retention purposes. + +## What is never deleted + +These are not configurable, because each one exists to prevent a specific bad outcome: + +- **Open findings.** An unresolved secret is not old news — it is a secret nobody has dealt with. +- **Findings an analyst decided about**: `false_positive`, `accepted_risk`, or a *manual* + resolution. Those are judgements. Letting one expire silently means the same finding returns on + the next scan as new, with nobody remembering it was already considered. Only `auto` + resolutions — an inference from absence, made by a scan — are eligible. +- **Suppressed findings.** The suppression is the reason the finding is not in anyone's list; + deleting the row would resurrect it as new on the next scan (ADR 0008). +- **An open finding's event trail**, however old. It is what an analyst reads to understand the + state in front of them, and an old open finding is exactly the one needing explanation. + +Deleting a finding does take its own `FindingEvent` rows with it — the trail describes that +finding, and orphaning it would leave rows nothing can interpret. + +## How it runs + +In the API's maintenance loop, under the same Postgres advisory lock as the scheduler, so one +replica purges however many are running. It is last in the round: it is the only job that can be +slow on a database that has never been purged, and nothing else waits on it. + +Deletion is batched. A round that hits `ICEBERG_RETENTION_BATCH_SIZE` simply continues on the next +beat, so the first purge after configuring a window cannot hold locks for minutes. + +To run one now — useful immediately after configuring a window, when you want to see the number +rather than discover it in the audit log: + +```bash +python -m iceberg_api retention-purge # in the api container +``` + +It prints the counts per table and, like the scheduled round, audits itself if it removed +anything. + +## Purges are audited + +Deleting evidence is an administrative action, so a round that removed anything writes an +`audit_event` with `action = retention.purged`, recording the counts **and the windows that +justified them** — the row still explains itself a year later when the settings have changed. + +A round that deleted nothing writes nothing: an audit log full of "deleted 0 rows" every minute is +one nobody reads. + +That audit row is itself subject to `ICEBERG_RETENTION_AUDIT_EVENTS_DAYS`, like every other. That +is deliberate rather than an oversight — a purge record exempt from retention would be a special +case hiding from the policy it enforces. If you need purge records to outlive the audit window, +ship them off-box: they are in the structured logs as `retention_purged`. + +## Choosing windows + +There is no default that is right for everyone, which is why there is no default. Some anchors: + +- **Do you need to prove a secret was remediated?** Then auto-resolved findings are the evidence + that it was, and the window should be at least as long as the period you might be asked about. +- **Regulatory audit-log windows** (SOX, PCI DSS, ISO 27001 and friends) typically land between + one and seven years. `ICEBERG_RETENTION_AUDIT_EVENTS_DAYS` should match whichever applies to + you; if you are unsure, leave it at `0` and ask, because you cannot get the rows back. +- **Storage pressure is rarely the real constraint.** A finding row is small. If the database is + growing uncomfortably it is usually `finding_event` on a noisy source, which + `ICEBERG_RETENTION_FINDING_EVENTS_DAYS` addresses without touching any finding. +- **Data-minimisation obligations** (GDPR Art. 5(1)(e) and similar) point the other way: keep + personal data no longer than necessary. Findings do not intentionally contain personal data — + the snippet is redacted (ADR 0004) — but a resource locator is a path in someone's wiki and can + name a person. If that matters for you, the findings window is the control. + +Whatever you choose, write it down somewhere other than the environment variable. The audit rows +record the window in force at the time of each purge, but only for purges that have already +happened. diff --git a/packages/core/src/iceberg_core/config.py b/packages/core/src/iceberg_core/config.py index 9a4fda2..9338593 100644 --- a/packages/core/src/iceberg_core/config.py +++ b/packages/core/src/iceberg_core/config.py @@ -165,6 +165,28 @@ class ApiSettings(SecretStoreSettings): #: a fresh identity per request. trusted_proxy_hops: int = Field(default=0, ge=0, le=10) + # ─── Data retention (#73) ───────────────────────────────────────────────── + # Findings and their append-only event trail grow forever otherwise. Every + # window is *off by default*: this database is evidence, and deleting some of + # it has to be a decision an operator made, not something that started + # happening because they upgraded (docs/retention.md). + # + # 0 means "keep forever" for all three. + # + #: Auto-resolved findings older than this are deleted. Only ``auto`` — a + #: finding an analyst resolved, accepted the risk on, or marked a false + #: positive is a decision, and decisions are not noise to be cleaned up. + retention_resolved_findings_days: int = Field(default=0, ge=0) + #: FindingEvent rows older than this are deleted, except the ones belonging to + #: findings that are still open. + retention_finding_events_days: int = Field(default=0, ge=0) + #: AuditEvent rows older than this are deleted. Usually the window a + #: compliance regime names; think before shortening it. + retention_audit_events_days: int = Field(default=0, ge=0) + #: Rows deleted per table per round, so one purge cannot lock a table for + #: minutes on a database that has never been purged before. + retention_batch_size: int = Field(default=1000, ge=1, le=100_000) + @field_validator("database_url", "redis_url") @classmethod def _require_url_scheme(cls, value: str) -> str: diff --git a/packages/core/src/iceberg_core/models/__init__.py b/packages/core/src/iceberg_core/models/__init__.py index 235f7d0..da635c5 100644 --- a/packages/core/src/iceberg_core/models/__init__.py +++ b/packages/core/src/iceberg_core/models/__init__.py @@ -18,6 +18,7 @@ AUDIT_CHANNEL_UPDATED, AUDIT_ENGINE_REGISTERED, AUDIT_ENGINE_TOKEN_ROTATED, + AUDIT_RETENTION_PURGED, AUDIT_SCHEDULE_CREATED, AUDIT_SCHEDULE_DELETED, AUDIT_SCHEDULE_UPDATED, @@ -30,6 +31,7 @@ AUDIT_SUPPRESSION_DELETED, AUDIT_TARGET_CHANNEL, AUDIT_TARGET_ENGINE, + AUDIT_TARGET_RETENTION, AUDIT_TARGET_SCHEDULE, AUDIT_TARGET_SOURCE, AUDIT_TARGET_SUPPRESSION, @@ -62,6 +64,7 @@ "AUDIT_CHANNEL_UPDATED", "AUDIT_ENGINE_REGISTERED", "AUDIT_ENGINE_TOKEN_ROTATED", + "AUDIT_RETENTION_PURGED", "AUDIT_SCHEDULE_CREATED", "AUDIT_SCHEDULE_DELETED", "AUDIT_SCHEDULE_UPDATED", @@ -74,6 +77,7 @@ "AUDIT_SUPPRESSION_DELETED", "AUDIT_TARGET_CHANNEL", "AUDIT_TARGET_ENGINE", + "AUDIT_TARGET_RETENTION", "AUDIT_TARGET_SCHEDULE", "AUDIT_TARGET_SOURCE", "AUDIT_TARGET_SUPPRESSION", diff --git a/packages/core/src/iceberg_core/models/audit.py b/packages/core/src/iceberg_core/models/audit.py index 4a8fd06..ce9242a 100644 --- a/packages/core/src/iceberg_core/models/audit.py +++ b/packages/core/src/iceberg_core/models/audit.py @@ -39,6 +39,9 @@ AUDIT_CHANNEL_UPDATED = "channel.updated" AUDIT_CHANNEL_DELETED = "channel.deleted" AUDIT_CHANNEL_SECRET_SET = "channel.secret_set" # noqa: S105 # an audit action name +#: Retention purges (#73). Deleting evidence is itself an administrative action, +#: so it is recorded — with counts, and with the window that justified it. +AUDIT_RETENTION_PURGED = "retention.purged" #: Values for ``target_type``. AUDIT_TARGET_USER = "user" @@ -47,6 +50,8 @@ AUDIT_TARGET_SUPPRESSION = "suppression" AUDIT_TARGET_ENGINE = "engine" AUDIT_TARGET_CHANNEL = "channel" +#: A purge is about the deployment, not about one row, so it has no target id. +AUDIT_TARGET_RETENTION = "retention" class AuditEvent(IcebergModel, table=True):