From 5dc86b3bca04d20d34eb5c7fa614ddea7fda7028 Mon Sep 17 00:00:00 2001 From: Pulipelly Shashank Reddy Date: Thu, 23 Jul 2026 11:59:27 +0530 Subject: [PATCH 1/3] feat(stg): add OTel semconv messaging.kafka.cluster.id relationship rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add EXT-SERVICE → KAFKATOPIC relationship synthesis rules for messaging.kafka.cluster.id (OTel incubating semconv) emitted by OTel instrumentation packages (Java #18978, Python #4727, JS #3577, .NET #4739). The existing rules in this file use kafka.cluster.id (a legacy attribute). The new rules cover the OTel-standard attribute name messaging.kafka.cluster.id so that spans from OTel-instrumented producers/consumers resolve to the correct KAFKATOPIC entity via the clusterId field in the KAFKATOPIC candidate lookup. Also add kafka.cluster.id tag to infra_kafkatopic_otel_jmxmetrics_stg, infra_kafkatopic_otel_jmxscraper_stg, and infra_kafkatopic_otel_prometheus_stg synthesis rules so that KAFKATOPIC entities from those receivers expose the cluster UUID needed by the candidate lookup. Assisted-by: Claude Sonnet 4.6 --- .../infra-kafkatopic/definition.stg.yml | 3 + .../EXT-SERVICE-to-KAFKATOPIC.stg.yml | 68 +++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/entity-types/infra-kafkatopic/definition.stg.yml b/entity-types/infra-kafkatopic/definition.stg.yml index e521685db..356879d3c 100644 --- a/entity-types/infra-kafkatopic/definition.stg.yml +++ b/entity-types/infra-kafkatopic/definition.stg.yml @@ -32,6 +32,7 @@ synthesis: tags: instrumentation.provider: kafka.cluster.name: + kafka.cluster.id: topic: - ruleName: infra_kafkatopic_otel_jmxscraper_stg @@ -59,6 +60,7 @@ synthesis: tags: instrumentation.provider: kafka.cluster.name: + kafka.cluster.id: topic: - ruleName: infra_kafkatopic_otel_kafkametrics_stg @@ -114,6 +116,7 @@ synthesis: tags: instrumentation.provider: kafka.cluster.name: + kafka.cluster.id: topic: # Log-based rule: OTel kafkareceiver logs carry topic, kafka.cluster.name, and producer.service.name. diff --git a/relationships/synthesis/EXT-SERVICE-to-KAFKATOPIC.stg.yml b/relationships/synthesis/EXT-SERVICE-to-KAFKATOPIC.stg.yml index faf76f019..167e8feb9 100644 --- a/relationships/synthesis/EXT-SERVICE-to-KAFKATOPIC.stg.yml +++ b/relationships/synthesis/EXT-SERVICE-to-KAFKATOPIC.stg.yml @@ -493,3 +493,71 @@ relationships: - field: topic attribute: messaging.destination.name + # OTel instrumentation producer span → KAFKATOPIC via messaging.kafka.cluster.id (OTel semconv) + - name: extServiceProducesOtelInstrKafkaTopicStg + version: "1" + origins: + - Distributed Tracing + - OpenTelemetry + conditions: + - attribute: eventType + anyOf: [ "Span" ] + - attribute: newrelic.source + anyOf: [ "api.traces.otlp" ] + - attribute: entity.type + anyOf: [ "SERVICE" ] + - attribute: span.kind + anyOf: [ "producer", "client" ] + - attribute: messaging.destination.name + present: true + - attribute: messaging.kafka.cluster.id + present: true + relationship: + expires: PT75M + relationshipType: PRODUCES + source: + extractGuid: + attribute: entity.guid + target: + lookupGuid: + candidateCategory: KAFKATOPIC + fields: + - field: clusterId + attribute: messaging.kafka.cluster.id + - field: topic + attribute: messaging.destination.name + + # OTel instrumentation consumer span → KAFKATOPIC via messaging.kafka.cluster.id (OTel semconv) + - name: extServiceConsumesOtelInstrKafkaTopicStg + version: "1" + origins: + - Distributed Tracing + - OpenTelemetry + conditions: + - attribute: eventType + anyOf: [ "Span" ] + - attribute: newrelic.source + anyOf: [ "api.traces.otlp" ] + - attribute: entity.type + anyOf: [ "SERVICE" ] + - attribute: span.kind + anyOf: [ "consumer", "server" ] + - attribute: messaging.destination.name + present: true + - attribute: messaging.kafka.cluster.id + present: true + relationship: + expires: PT75M + relationshipType: CONSUMES + source: + extractGuid: + attribute: entity.guid + target: + lookupGuid: + candidateCategory: KAFKATOPIC + fields: + - field: clusterId + attribute: messaging.kafka.cluster.id + - field: topic + attribute: messaging.destination.name + From c1f01d7f0efcd0ccab3d0dd4318b50f4cd47efc9 Mon Sep 17 00:00:00 2001 From: Pulipelly Shashank Reddy Date: Thu, 23 Jul 2026 12:22:30 +0530 Subject: [PATCH 2/3] fix(stg): correct span attribute in ClusterId relationship rules The existing extServiceProducesSelfHostKafkaTopicClusterIdStg and extServiceConsumesSelfHostKafkaTopicClusterIdStg rules matched kafka.cluster.id on spans, which is not a real OTel span attribute. OTel instrumentation packages (Java, Python, JS, .NET) emit messaging.kafka.cluster.id per OTel incubating semconv. Update both rules to match messaging.kafka.cluster.id and remove the duplicate rules added in the previous commit. Assisted-by: Claude Sonnet 4.6 --- .../EXT-SERVICE-to-KAFKATOPIC.stg.yml | 74 +------------------ 1 file changed, 3 insertions(+), 71 deletions(-) diff --git a/relationships/synthesis/EXT-SERVICE-to-KAFKATOPIC.stg.yml b/relationships/synthesis/EXT-SERVICE-to-KAFKATOPIC.stg.yml index 167e8feb9..ca30b4090 100644 --- a/relationships/synthesis/EXT-SERVICE-to-KAFKATOPIC.stg.yml +++ b/relationships/synthesis/EXT-SERVICE-to-KAFKATOPIC.stg.yml @@ -425,76 +425,8 @@ relationships: - field: producerServiceName attribute: producer.service.name - # Self-hosted Kafka producer span → KAFKATOPIC via kafka.cluster.id + messaging.destination.name + # OTel producer span → KAFKATOPIC via messaging.kafka.cluster.id (OTel incubating semconv) - name: extServiceProducesSelfHostKafkaTopicClusterIdStg - version: "1" - origins: - - Distributed Tracing - - OpenTelemetry - conditions: - - attribute: eventType - anyOf: [ "Span" ] - - attribute: newrelic.source - anyOf: [ "api.traces.otlp" ] - - attribute: entity.type - anyOf: [ "SERVICE" ] - - attribute: span.kind - anyOf: [ "producer", "client" ] - - attribute: messaging.destination.name - present: true - - attribute: kafka.cluster.id - present: true - relationship: - expires: PT75M - relationshipType: PRODUCES - source: - extractGuid: - attribute: entity.guid - target: - lookupGuid: - candidateCategory: KAFKATOPIC - fields: - - field: clusterId - attribute: kafka.cluster.id - - field: topic - attribute: messaging.destination.name - - # Self-hosted Kafka consumer span → KAFKATOPIC via kafka.cluster.id + messaging.destination.name - - name: extServiceConsumesSelfHostKafkaTopicClusterIdStg - version: "1" - origins: - - Distributed Tracing - - OpenTelemetry - conditions: - - attribute: eventType - anyOf: [ "Span" ] - - attribute: newrelic.source - anyOf: [ "api.traces.otlp" ] - - attribute: entity.type - anyOf: [ "SERVICE" ] - - attribute: span.kind - anyOf: [ "consumer", "server" ] - - attribute: messaging.destination.name - present: true - - attribute: kafka.cluster.id - present: true - relationship: - expires: PT75M - relationshipType: CONSUMES - source: - extractGuid: - attribute: entity.guid - target: - lookupGuid: - candidateCategory: KAFKATOPIC - fields: - - field: clusterId - attribute: kafka.cluster.id - - field: topic - attribute: messaging.destination.name - - # OTel instrumentation producer span → KAFKATOPIC via messaging.kafka.cluster.id (OTel semconv) - - name: extServiceProducesOtelInstrKafkaTopicStg version: "1" origins: - Distributed Tracing @@ -527,8 +459,8 @@ relationships: - field: topic attribute: messaging.destination.name - # OTel instrumentation consumer span → KAFKATOPIC via messaging.kafka.cluster.id (OTel semconv) - - name: extServiceConsumesOtelInstrKafkaTopicStg + # OTel consumer span → KAFKATOPIC via messaging.kafka.cluster.id (OTel incubating semconv) + - name: extServiceConsumesSelfHostKafkaTopicClusterIdStg version: "1" origins: - Distributed Tracing From 6d56a363eaa6261ae1ad564f0512f3d27eb58fb5 Mon Sep 17 00:00:00 2001 From: Pulipelly Shashank Reddy Date: Thu, 23 Jul 2026 12:42:27 +0530 Subject: [PATCH 3/3] feat(stg): add metric-based relationship rules for messaging.kafka.cluster.id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add two STG relationship rules for Java kafka-clients-cluster-metrics metrics that carry messaging.kafka.cluster.id (OTel incubating semconv) as a data-point attribute: - extServiceProducesKafkaClientMetricsTopicClusterIdStg (kafka.producer.* metrics) - extServiceConsumesKafkaClientMetricsTopicClusterIdStg (kafka.consumer.* metrics) Both rules use lookupGuid(KAFKATOPIC, clusterId=messaging.kafka.cluster.id, topic=topic) matching the existing KAFKATOPIC candidates file which maps clusterId → kafka.cluster.id tag. This covers the app-side metrics path from opentelemetry-java-instrumentation PR #18978 which adds messaging.kafka.cluster.id to kafka-clients-cluster-metrics via ClusterResourceListener.onUpdate(). The collector-side (kafkametricsreceiver PR #48897) emits kafka.cluster.id as a resource attribute, handled by existing KAFKATOPIC synthesis tags added in the earlier commit. Assisted-by: Claude Sonnet 4.6 --- .../EXT-SERVICE-to-KAFKATOPIC.stg.yml | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/relationships/synthesis/EXT-SERVICE-to-KAFKATOPIC.stg.yml b/relationships/synthesis/EXT-SERVICE-to-KAFKATOPIC.stg.yml index ca30b4090..768a05a0e 100644 --- a/relationships/synthesis/EXT-SERVICE-to-KAFKATOPIC.stg.yml +++ b/relationships/synthesis/EXT-SERVICE-to-KAFKATOPIC.stg.yml @@ -493,3 +493,69 @@ relationships: - field: topic attribute: messaging.destination.name + # Java kafka-clients-cluster-metrics producer metrics → KAFKATOPIC via messaging.kafka.cluster.id + topic + - name: extServiceProducesKafkaClientMetricsTopicClusterIdStg + version: "1" + origins: + - OpenTelemetry + conditions: + - attribute: eventType + anyOf: [ "Metric" ] + - attribute: instrumentation.provider + anyOf: [ "opentelemetry" ] + - attribute: metricName + startsWith: "kafka.producer" + - attribute: entity.type + anyOf: [ "SERVICE" ] + - attribute: messaging.kafka.cluster.id + present: true + - attribute: topic + present: true + relationship: + expires: PT75M + relationshipType: PRODUCES + source: + extractGuid: + attribute: entity.guid + target: + lookupGuid: + candidateCategory: KAFKATOPIC + fields: + - field: clusterId + attribute: messaging.kafka.cluster.id + - field: topic + attribute: topic + + # Java kafka-clients-cluster-metrics consumer metrics → KAFKATOPIC via messaging.kafka.cluster.id + topic + - name: extServiceConsumesKafkaClientMetricsTopicClusterIdStg + version: "1" + origins: + - OpenTelemetry + conditions: + - attribute: eventType + anyOf: [ "Metric" ] + - attribute: instrumentation.provider + anyOf: [ "opentelemetry" ] + - attribute: metricName + startsWith: "kafka.consumer" + - attribute: entity.type + anyOf: [ "SERVICE" ] + - attribute: messaging.kafka.cluster.id + present: true + - attribute: topic + present: true + relationship: + expires: PT75M + relationshipType: CONSUMES + source: + extractGuid: + attribute: entity.guid + target: + lookupGuid: + candidateCategory: KAFKATOPIC + fields: + - field: clusterId + attribute: messaging.kafka.cluster.id + - field: topic + attribute: topic +