From 90f9e72d148271a1ac39c61d3f587e2d2a1c3d6c Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Fri, 26 Jun 2026 23:07:09 -0400 Subject: [PATCH 1/3] fix: stop gateway ActivityPolicy DLQ leak on rejected creates The create rule's summary dereferenced audit.responseObject.metadata.name, but a rejected create (e.g. 409/422) returns a Status object with no metadata.name. The rule still matched (it only checks requestObject.spec), so CEL evaluation raised "no such key: name", the event went to the DLQ, and retries failed identically -- a slow, steady DLQ leak (DLQSlowLeak in prod). Guard the leaf path and fall back to objectRef.name, preserving the server-assigned-name display for successful creates while staying null-safe for rejected ones. Fixes the prod DLQSlowLeak on policy gateway.networking.k8s.io-gateway (error_type=cel_summary). --- config/milo/activity/policies/gateway-policy.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/milo/activity/policies/gateway-policy.yaml b/config/milo/activity/policies/gateway-policy.yaml index 1cdcbb52..13a330eb 100644 --- a/config/milo/activity/policies/gateway-policy.yaml +++ b/config/milo/activity/policies/gateway-policy.yaml @@ -15,9 +15,11 @@ spec: auditRules: # Gateway creation with spec available + # responseObject lacks metadata.name when a create is rejected (the apiserver + # returns a Status object), so guard the leaf and fall back to objectRef.name. - name: create match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec)" - summary: "{{ actor }} created gateway {{ link(audit.responseObject.metadata.name, audit.objectRef) }}" + summary: "{{ actor }} created gateway {{ has(audit.responseObject.metadata.name) ? link(audit.responseObject.metadata.name, audit.objectRef) : link(audit.objectRef.name, audit.objectRef) }}" # Gateway creation fallback (no spec) - name: create-fallback From 7edfe16fbd9512a674228420ef608a0f54178787 Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Fri, 26 Jun 2026 23:15:53 -0400 Subject: [PATCH 2/3] fix: null-guard responseObject derefs in all gateway-domain ActivityPolicies Same DLQ-leak class as the gateway create rule: create/update summaries dereference audit.responseObject.{metadata.name,spec.domainName}, but a rejected create/update returns a Status object with no spec/metadata.name, so CEL raises "no such key" and the event leaks to the DLQ. Guard the leaf and fall back to audit.objectRef.name (always present) across: - backendtlspolicy, connector, connectoradvertisement, trafficprotection (create) - domain (create + update) delete rules already guard has(audit.responseObject.spec) and are unchanged. --- config/milo/activity/policies/backendtlspolicy-policy.yaml | 2 +- config/milo/activity/policies/connector-policy.yaml | 2 +- .../milo/activity/policies/connectoradvertisement-policy.yaml | 2 +- config/milo/activity/policies/domain-policy.yaml | 4 ++-- .../activity/policies/trafficprotectionpolicy-policy.yaml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/milo/activity/policies/backendtlspolicy-policy.yaml b/config/milo/activity/policies/backendtlspolicy-policy.yaml index 240e8996..78d6ff55 100644 --- a/config/milo/activity/policies/backendtlspolicy-policy.yaml +++ b/config/milo/activity/policies/backendtlspolicy-policy.yaml @@ -17,7 +17,7 @@ spec: # BackendTLSPolicy creation with spec available - name: create match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec)" - summary: "{{ actor }} created backend TLS policy {{ link(audit.responseObject.metadata.name, audit.objectRef) }}" + summary: "{{ actor }} created backend TLS policy {{ has(audit.responseObject.metadata.name) ? link(audit.responseObject.metadata.name, audit.objectRef) : link(audit.objectRef.name, audit.objectRef) }}" # BackendTLSPolicy creation fallback (no spec) - name: create-fallback diff --git a/config/milo/activity/policies/connector-policy.yaml b/config/milo/activity/policies/connector-policy.yaml index b8528eae..e6bcecd8 100644 --- a/config/milo/activity/policies/connector-policy.yaml +++ b/config/milo/activity/policies/connector-policy.yaml @@ -17,7 +17,7 @@ spec: # Connector creation with spec available - name: create match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec)" - summary: "{{ actor }} created connector {{ link(audit.responseObject.metadata.name, audit.objectRef) }}" + summary: "{{ actor }} created connector {{ has(audit.responseObject.metadata.name) ? link(audit.responseObject.metadata.name, audit.objectRef) : link(audit.objectRef.name, audit.objectRef) }}" # Connector creation fallback (no spec) - name: create-fallback diff --git a/config/milo/activity/policies/connectoradvertisement-policy.yaml b/config/milo/activity/policies/connectoradvertisement-policy.yaml index e7e40a77..20d6d094 100644 --- a/config/milo/activity/policies/connectoradvertisement-policy.yaml +++ b/config/milo/activity/policies/connectoradvertisement-policy.yaml @@ -18,7 +18,7 @@ spec: # ConnectorAdvertisement creation with spec available - name: create match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec)" - summary: "{{ actor }} created connector advertisement {{ link(audit.responseObject.metadata.name, audit.objectRef) }}" + summary: "{{ actor }} created connector advertisement {{ has(audit.responseObject.metadata.name) ? link(audit.responseObject.metadata.name, audit.objectRef) : link(audit.objectRef.name, audit.objectRef) }}" # ConnectorAdvertisement creation fallback (no spec) - name: create-fallback diff --git a/config/milo/activity/policies/domain-policy.yaml b/config/milo/activity/policies/domain-policy.yaml index b56e209f..745b48ea 100644 --- a/config/milo/activity/policies/domain-policy.yaml +++ b/config/milo/activity/policies/domain-policy.yaml @@ -18,7 +18,7 @@ spec: # Domain creation with spec.domainName available - use domain name as display text - name: create match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec)" - summary: "{{ actor }} created domain {{ link(audit.responseObject.spec.domainName, audit.objectRef) }}" + summary: "{{ actor }} created domain {{ has(audit.responseObject.spec.domainName) ? link(audit.responseObject.spec.domainName, audit.objectRef) : link(audit.objectRef.name, audit.objectRef) }}" # Domain creation fallback (no spec) - name: create-fallback @@ -38,7 +38,7 @@ spec: # Domain update with spec available - excludes status subresource - name: update match: "!audit.user.username.startsWith('system:') && audit.verb in ['update', 'patch'] && !has(audit.objectRef.subresource) && has(audit.requestObject.spec)" - summary: "{{ actor }} updated domain {{ link(audit.responseObject.spec.domainName, audit.objectRef) }}" + summary: "{{ actor }} updated domain {{ has(audit.responseObject.spec.domainName) ? link(audit.responseObject.spec.domainName, audit.objectRef) : link(audit.objectRef.name, audit.objectRef) }}" # Domain update fallback (no spec) - name: update-fallback diff --git a/config/milo/activity/policies/trafficprotectionpolicy-policy.yaml b/config/milo/activity/policies/trafficprotectionpolicy-policy.yaml index adb19e2e..66d66ed4 100644 --- a/config/milo/activity/policies/trafficprotectionpolicy-policy.yaml +++ b/config/milo/activity/policies/trafficprotectionpolicy-policy.yaml @@ -17,7 +17,7 @@ spec: # TrafficProtectionPolicy creation with spec available - name: create match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec)" - summary: "{{ actor }} created traffic protection policy {{ link(audit.responseObject.metadata.name, audit.objectRef) }}" + summary: "{{ actor }} created traffic protection policy {{ has(audit.responseObject.metadata.name) ? link(audit.responseObject.metadata.name, audit.objectRef) : link(audit.objectRef.name, audit.objectRef) }}" # TrafficProtectionPolicy creation fallback (no spec) - name: create-fallback From e639936382f06064b94a26dc7dd08495b5e5714d Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Sat, 27 Jun 2026 00:37:52 -0400 Subject: [PATCH 3/3] fix: extend ActivityPolicy summary fallback for rejected generateName creates The earlier fix guarded responseObject. and fell back to audit.objectRef.name, but a live audit event from staging exposed two gaps: 1. generateName creates (connector, connectoradvertisement) rejected before a name is assigned have an EMPTY audit.objectRef.name. The fallback then derefs audit.objectRef.name on a Status responseObject and raises "no such key: name" again, so the event still dead-letters. The name is carried on the Status at responseObject.details.name. 2. domain create/update used has(audit.responseObject.spec.domainName), but nested has(a.b.c) evaluates the intermediate selection and errors when a rejected Status has no spec. This is the same DLQ bug class for every rejected domain create/update. Replace each create/update summary with a per-level-guarded fallback chain: responseObject. -> objectRef.name -> responseObject.details.name -> literal The details branch is guarded with has(responseObject.details) because some Status responses carry no details. domain also guards has(responseObject.spec). Evidence: activity-processor logs (processor.go:945) on staging show networking.datumapis.com-connector events re-failing the DLQ retry (retryCount climbing) on the previously "fixed" objectRef.name branch. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../activity/policies/backendtlspolicy-policy.yaml | 7 +++++-- config/milo/activity/policies/connector-policy.yaml | 9 +++++++-- .../policies/connectoradvertisement-policy.yaml | 7 +++++-- config/milo/activity/policies/domain-policy.yaml | 12 ++++++++---- config/milo/activity/policies/gateway-policy.yaml | 8 +++++--- .../policies/trafficprotectionpolicy-policy.yaml | 7 +++++-- 6 files changed, 35 insertions(+), 15 deletions(-) diff --git a/config/milo/activity/policies/backendtlspolicy-policy.yaml b/config/milo/activity/policies/backendtlspolicy-policy.yaml index 78d6ff55..e7fe5588 100644 --- a/config/milo/activity/policies/backendtlspolicy-policy.yaml +++ b/config/milo/activity/policies/backendtlspolicy-policy.yaml @@ -14,10 +14,13 @@ spec: kind: BackendTLSPolicy auditRules: - # BackendTLSPolicy creation with spec available + # BackendTLSPolicy creation with spec available. + # Rejected creates return a Status responseObject (no metadata.name); for + # generateName creates objectRef.name is also empty, so fall back to + # responseObject.details.name. has() is guarded per level. - name: create match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec)" - summary: "{{ actor }} created backend TLS policy {{ has(audit.responseObject.metadata.name) ? link(audit.responseObject.metadata.name, audit.objectRef) : link(audit.objectRef.name, audit.objectRef) }}" + summary: "{{ actor }} created backend TLS policy {{ has(audit.responseObject.metadata.name) ? link(audit.responseObject.metadata.name, audit.objectRef) : has(audit.objectRef.name) ? link(audit.objectRef.name, audit.objectRef) : (has(audit.responseObject.details) && has(audit.responseObject.details.name)) ? link(audit.responseObject.details.name, audit.objectRef) : link('a backend TLS policy', audit.objectRef) }}" # BackendTLSPolicy creation fallback (no spec) - name: create-fallback diff --git a/config/milo/activity/policies/connector-policy.yaml b/config/milo/activity/policies/connector-policy.yaml index e6bcecd8..39f5a5bc 100644 --- a/config/milo/activity/policies/connector-policy.yaml +++ b/config/milo/activity/policies/connector-policy.yaml @@ -14,10 +14,15 @@ spec: kind: Connector auditRules: - # Connector creation with spec available + # Connector creation with spec available. + # Clients create Connectors via generateName (e.g. "datum-connect-"). A rejected + # create (quota/admission) returns a Status as responseObject with no metadata.name, + # and objectRef.name is empty because no name was assigned — so fall back to + # responseObject.details.name. has() is guarded per level (nested has errors on a + # missing intermediate). - name: create match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec)" - summary: "{{ actor }} created connector {{ has(audit.responseObject.metadata.name) ? link(audit.responseObject.metadata.name, audit.objectRef) : link(audit.objectRef.name, audit.objectRef) }}" + summary: "{{ actor }} created connector {{ has(audit.responseObject.metadata.name) ? link(audit.responseObject.metadata.name, audit.objectRef) : has(audit.objectRef.name) ? link(audit.objectRef.name, audit.objectRef) : (has(audit.responseObject.details) && has(audit.responseObject.details.name)) ? link(audit.responseObject.details.name, audit.objectRef) : link('a connector', audit.objectRef) }}" # Connector creation fallback (no spec) - name: create-fallback diff --git a/config/milo/activity/policies/connectoradvertisement-policy.yaml b/config/milo/activity/policies/connectoradvertisement-policy.yaml index 20d6d094..1ee165e6 100644 --- a/config/milo/activity/policies/connectoradvertisement-policy.yaml +++ b/config/milo/activity/policies/connectoradvertisement-policy.yaml @@ -15,10 +15,13 @@ spec: kind: ConnectorAdvertisement auditRules: - # ConnectorAdvertisement creation with spec available + # ConnectorAdvertisement creation with spec available. + # Like Connector, advertisements may be created via generateName; a rejected create + # yields a Status responseObject (no metadata.name) with an empty objectRef.name, so + # fall back to responseObject.details.name. has() is guarded per level. - name: create match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec)" - summary: "{{ actor }} created connector advertisement {{ has(audit.responseObject.metadata.name) ? link(audit.responseObject.metadata.name, audit.objectRef) : link(audit.objectRef.name, audit.objectRef) }}" + summary: "{{ actor }} created connector advertisement {{ has(audit.responseObject.metadata.name) ? link(audit.responseObject.metadata.name, audit.objectRef) : has(audit.objectRef.name) ? link(audit.objectRef.name, audit.objectRef) : (has(audit.responseObject.details) && has(audit.responseObject.details.name)) ? link(audit.responseObject.details.name, audit.objectRef) : link('a connector advertisement', audit.objectRef) }}" # ConnectorAdvertisement creation fallback (no spec) - name: create-fallback diff --git a/config/milo/activity/policies/domain-policy.yaml b/config/milo/activity/policies/domain-policy.yaml index 745b48ea..b5f42043 100644 --- a/config/milo/activity/policies/domain-policy.yaml +++ b/config/milo/activity/policies/domain-policy.yaml @@ -15,10 +15,13 @@ spec: kind: Domain auditRules: - # Domain creation with spec.domainName available - use domain name as display text + # Domain creation with spec.domainName available - use domain name as display text. + # A rejected create returns a Status responseObject (no spec), so has() must guard + # the spec level too: nested has(responseObject.spec.domainName) errors when spec is + # absent. Fall back to objectRef.name, then responseObject.details.name (generateName). - name: create match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec)" - summary: "{{ actor }} created domain {{ has(audit.responseObject.spec.domainName) ? link(audit.responseObject.spec.domainName, audit.objectRef) : link(audit.objectRef.name, audit.objectRef) }}" + summary: "{{ actor }} created domain {{ (has(audit.responseObject.spec) && has(audit.responseObject.spec.domainName)) ? link(audit.responseObject.spec.domainName, audit.objectRef) : has(audit.objectRef.name) ? link(audit.objectRef.name, audit.objectRef) : (has(audit.responseObject.details) && has(audit.responseObject.details.name)) ? link(audit.responseObject.details.name, audit.objectRef) : link('a domain', audit.objectRef) }}" # Domain creation fallback (no spec) - name: create-fallback @@ -35,10 +38,11 @@ spec: match: "!audit.user.username.startsWith('system:') && audit.verb == 'delete'" summary: "{{ actor }} deleted a domain" - # Domain update with spec available - excludes status subresource + # Domain update with spec available - excludes status subresource. + # Same Status-response guarding as create: a rejected update has no responseObject.spec. - name: update match: "!audit.user.username.startsWith('system:') && audit.verb in ['update', 'patch'] && !has(audit.objectRef.subresource) && has(audit.requestObject.spec)" - summary: "{{ actor }} updated domain {{ has(audit.responseObject.spec.domainName) ? link(audit.responseObject.spec.domainName, audit.objectRef) : link(audit.objectRef.name, audit.objectRef) }}" + summary: "{{ actor }} updated domain {{ (has(audit.responseObject.spec) && has(audit.responseObject.spec.domainName)) ? link(audit.responseObject.spec.domainName, audit.objectRef) : has(audit.objectRef.name) ? link(audit.objectRef.name, audit.objectRef) : (has(audit.responseObject.details) && has(audit.responseObject.details.name)) ? link(audit.responseObject.details.name, audit.objectRef) : link('a domain', audit.objectRef) }}" # Domain update fallback (no spec) - name: update-fallback diff --git a/config/milo/activity/policies/gateway-policy.yaml b/config/milo/activity/policies/gateway-policy.yaml index 13a330eb..982cdbf7 100644 --- a/config/milo/activity/policies/gateway-policy.yaml +++ b/config/milo/activity/policies/gateway-policy.yaml @@ -15,11 +15,13 @@ spec: auditRules: # Gateway creation with spec available - # responseObject lacks metadata.name when a create is rejected (the apiserver - # returns a Status object), so guard the leaf and fall back to objectRef.name. + # A rejected create returns a Status object as responseObject (no metadata.name). + # For generateName creates rejected before naming, objectRef.name is also empty, + # so fall back further to responseObject.details.name (carried on the Status). + # has() is guarded at each level: nested has(a.b.c) errors if b is absent. - name: create match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec)" - summary: "{{ actor }} created gateway {{ has(audit.responseObject.metadata.name) ? link(audit.responseObject.metadata.name, audit.objectRef) : link(audit.objectRef.name, audit.objectRef) }}" + summary: "{{ actor }} created gateway {{ has(audit.responseObject.metadata.name) ? link(audit.responseObject.metadata.name, audit.objectRef) : has(audit.objectRef.name) ? link(audit.objectRef.name, audit.objectRef) : (has(audit.responseObject.details) && has(audit.responseObject.details.name)) ? link(audit.responseObject.details.name, audit.objectRef) : link('a gateway', audit.objectRef) }}" # Gateway creation fallback (no spec) - name: create-fallback diff --git a/config/milo/activity/policies/trafficprotectionpolicy-policy.yaml b/config/milo/activity/policies/trafficprotectionpolicy-policy.yaml index 66d66ed4..6575e972 100644 --- a/config/milo/activity/policies/trafficprotectionpolicy-policy.yaml +++ b/config/milo/activity/policies/trafficprotectionpolicy-policy.yaml @@ -14,10 +14,13 @@ spec: kind: TrafficProtectionPolicy auditRules: - # TrafficProtectionPolicy creation with spec available + # TrafficProtectionPolicy creation with spec available. + # Rejected creates return a Status responseObject (no metadata.name); for + # generateName creates objectRef.name is also empty, so fall back to + # responseObject.details.name. has() is guarded per level. - name: create match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec)" - summary: "{{ actor }} created traffic protection policy {{ has(audit.responseObject.metadata.name) ? link(audit.responseObject.metadata.name, audit.objectRef) : link(audit.objectRef.name, audit.objectRef) }}" + summary: "{{ actor }} created traffic protection policy {{ has(audit.responseObject.metadata.name) ? link(audit.responseObject.metadata.name, audit.objectRef) : has(audit.objectRef.name) ? link(audit.objectRef.name, audit.objectRef) : (has(audit.responseObject.details) && has(audit.responseObject.details.name)) ? link(audit.responseObject.details.name, audit.objectRef) : link('a traffic protection policy', audit.objectRef) }}" # TrafficProtectionPolicy creation fallback (no spec) - name: create-fallback