From cdc57078f84ec58909ea6f7710f1b45a1246d762 Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Mon, 11 May 2026 14:42:15 +0300 Subject: [PATCH 1/5] Remove summaries from updated fields in patient CDC events --- .../clinic/templates/5-patients-source-kafka-connector.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/tidepool/charts/clinic/templates/5-patients-source-kafka-connector.yaml b/charts/tidepool/charts/clinic/templates/5-patients-source-kafka-connector.yaml index 6857b9b76..98cf3ea3f 100644 --- a/charts/tidepool/charts/clinic/templates/5-patients-source-kafka-connector.yaml +++ b/charts/tidepool/charts/clinic/templates/5-patients-source-kafka-connector.yaml @@ -14,8 +14,8 @@ spec: collection: patients connection.uri: {{ .Values.global.kafka.connect.mongoConnectionUri }} copy.existing: false - pipeline: '[ {$project: {"fullDocument.summary": 0}} ]' - startup.mode.copy.existing.pipeline: '[ {$project: {"fullDocument.summary": 0}} ]' + pipeline: '[ {$project: {"fullDocument.summary": 0, "updateDescription.updatedFields.summary": 0}} ]' + startup.mode.copy.existing.pipeline: '[ {$project: {"fullDocument.summary": 0, "updateDescription.updatedFields.summary": 0}} ]' database: clinic key.converter: org.apache.kafka.connect.json.JsonConverter key.converter.schemas.enable: false From 6a4bf9ba8a991f79a82a69ae2e56d948300a3309 Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Wed, 10 Jun 2026 11:50:34 +0300 Subject: [PATCH 2/5] Add user activity table --- .../templates/2-users-source-kafka-connector.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml b/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml index 85717257c..33015bcff 100644 --- a/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml +++ b/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml @@ -27,11 +27,17 @@ spec: value.converter: 'org.apache.kafka.connect.json.JsonConverter' value.converter.schemas.enable: false snapshot.mode: {{ .Values.keycloak.snapshotMode }} - table.include.list: 'public.user_entity,public.user_role_mapping,public.keycloak_role,public.user_attribute' - transforms: 'filter,filter_user_attr' + table.include.list: 'public.user_entity,public.user_role_mapping,public.keycloak_role,public.user_attribute,public.tidepool_user_activity_event' + # Key the user-activity outbox by user_id (rather than the row's PK) so every + # event for a user lands on one partition and is consumed in commit order. This + # is what lets the clinic-worker consumer apply updates as last-writer-wins + # safely; combined with snapshot.mode=never it removes out-of-order/replay + # regressions. Other tables keep their default primary-key based message key. + message.key.columns: 'public.tidepool_user_activity_event:user_id' + transforms: 'filter' transforms.filter.type: 'io.debezium.transforms.Filter' transforms.filter.language: 'jsr223.groovy' - transforms.filter.topic.regex: '.+\.public.(user_entity|keycloak_role)' + transforms.filter.topic.regex: '.+\.public.(user_entity|keycloak_role|tidepool_user_activity_event)' transforms.filter.condition: "value.op && (((value.op == 'r' || value.op == 'c' || value.op == 'u') && (value.after && value.after.realm_id && value.after.realm_id == '{{ $realm }}')) || (value.op == 'd'))" transforms.filter_user_attr.type: 'io.debezium.transforms.Filter' transforms.filter_user_attr.language: 'jsr223.groovy' From 82350e651f1d0ca7ec57f94e34ff15d7653a2905 Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Wed, 10 Jun 2026 12:03:17 +0300 Subject: [PATCH 3/5] Add user activity events --- .../templates/2-users-source-kafka-connector.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml b/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml index 33015bcff..e889efc06 100644 --- a/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml +++ b/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml @@ -34,11 +34,20 @@ spec: # safely; combined with snapshot.mode=never it removes out-of-order/replay # regressions. Other tables keep their default primary-key based message key. message.key.columns: 'public.tidepool_user_activity_event:user_id' - transforms: 'filter' + transforms: 'filter,user_activity_filter,filter_user_attr' transforms.filter.type: 'io.debezium.transforms.Filter' transforms.filter.language: 'jsr223.groovy' - transforms.filter.topic.regex: '.+\.public.(user_entity|keycloak_role|tidepool_user_activity_event)' + transforms.filter.topic.regex: '.+\.public.(user_entity|keycloak_role)' transforms.filter.condition: "value.op && (((value.op == 'r' || value.op == 'c' || value.op == 'u') && (value.after && value.after.realm_id && value.after.realm_id == '{{ $realm }}')) || (value.op == 'd'))" + # Realm filter scoped to the user-activity outbox only. Keeps inserts (and any + # snapshot reads/updates) for the configured realm and drops everything else, + # including pruning deletes (their `after` is null, so they fail the check) — + # the clinic-worker consumer ignores deletes anyway. Records on other topics do + # not match this regex and pass through untouched. + transforms.user_activity_filter.type: 'io.debezium.transforms.Filter' + transforms.user_activity_filter.language: 'jsr223.groovy' + transforms.user_activity_filter.topic.regex: '.+\.public.tidepool_user_activity_event' + transforms.user_activity_filter.condition: "value.op && (value.op == 'r' || value.op == 'c' || value.op == 'u') && value.after && value.after.realm_id && value.after.realm_id == '{{ $realm }}'" transforms.filter_user_attr.type: 'io.debezium.transforms.Filter' transforms.filter_user_attr.language: 'jsr223.groovy' transforms.filter_user_attr.topic.regex: '.+\.public.(user_attribute)' From 2ceeef3115cdb8e9fa7198d6050a1eac7b6b1635 Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Thu, 11 Jun 2026 17:22:05 +0300 Subject: [PATCH 4/5] Drop user-activity tombstones in connector filter Set null.handling.mode=drop on user_activity_filter so pruning-delete tombstones are discarded at the source instead of reaching the consumer. Scoped to this filter's topic.regex, so keycloak tables are unaffected. --- .../kafka/templates/2-users-source-kafka-connector.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml b/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml index e889efc06..016741ec9 100644 --- a/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml +++ b/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml @@ -47,6 +47,10 @@ spec: transforms.user_activity_filter.type: 'io.debezium.transforms.Filter' transforms.user_activity_filter.language: 'jsr223.groovy' transforms.user_activity_filter.topic.regex: '.+\.public.tidepool_user_activity_event' + # Drop pruning-delete tombstones (null value) for this topic instead of the + # default 'keep', so they don't reach the consumer. Scoped to this filter's + # topic.regex, so the keycloak tables' tombstones are unaffected. + transforms.user_activity_filter.null.handling.mode: 'drop' transforms.user_activity_filter.condition: "value.op && (value.op == 'r' || value.op == 'c' || value.op == 'u') && value.after && value.after.realm_id && value.after.realm_id == '{{ $realm }}'" transforms.filter_user_attr.type: 'io.debezium.transforms.Filter' transforms.filter_user_attr.language: 'jsr223.groovy' From f1346de8fc75de8df046755d45e64ced12208b86 Mon Sep 17 00:00:00 2001 From: Todd Kazakov Date: Fri, 14 Aug 2026 16:38:52 +0300 Subject: [PATCH 5/5] Allow specifying custom debezium slot name --- .../charts/kafka/templates/2-users-source-kafka-connector.yaml | 1 + charts/tidepool/values.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml b/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml index 016741ec9..dbca38a9d 100644 --- a/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml +++ b/charts/tidepool/charts/kafka/templates/2-users-source-kafka-connector.yaml @@ -27,6 +27,7 @@ spec: value.converter: 'org.apache.kafka.connect.json.JsonConverter' value.converter.schemas.enable: false snapshot.mode: {{ .Values.keycloak.snapshotMode }} + slot.name: {{ .Values.global.kafka.connect.keycloak.slotName }} table.include.list: 'public.user_entity,public.user_role_mapping,public.keycloak_role,public.user_attribute,public.tidepool_user_activity_event' # Key the user-activity outbox by user_id (rather than the row's PK) so every # event for a user lands on one partition and is consumed in commit order. This diff --git a/charts/tidepool/values.yaml b/charts/tidepool/values.yaml index 43f9c7c4e..68f6a00df 100644 --- a/charts/tidepool/values.yaml +++ b/charts/tidepool/values.yaml @@ -51,6 +51,7 @@ global: # -- enable keycloak kafka connector keycloak: enabled: false + slotName: debezium # -- The password key ref passwordKeyName: "Password" linkerdsupport: