From 0764fc6011a854bca63a9fdec9b7dc3be9b1ae25 Mon Sep 17 00:00:00 2001 From: Nitish Date: Mon, 4 May 2026 18:58:44 +0530 Subject: [PATCH 1/4] Removed additional internal metrics in services --- internal/supervisor/collector_config.yaml.tmpl | 9 --------- 1 file changed, 9 deletions(-) diff --git a/internal/supervisor/collector_config.yaml.tmpl b/internal/supervisor/collector_config.yaml.tmpl index 3410487..0e4c820 100644 --- a/internal/supervisor/collector_config.yaml.tmpl +++ b/internal/supervisor/collector_config.yaml.tmpl @@ -53,15 +53,6 @@ exporters: max_elapsed_time: 120s service: - telemetry: - metrics: - # Internal metrics on loopback only (replaces deprecated metrics.address for otelcol 0.130+). - readers: - - pull: - exporter: - prometheus: - host: 127.0.0.1 - port: 9464 pipelines: logs: receivers: [journald] From 4351cf3126f219663a7cf06d59439bbf097c2a7e Mon Sep 17 00:00:00 2001 From: Nitish Date: Fri, 8 May 2026 17:01:17 +0530 Subject: [PATCH 2/4] Added internal telemetry changes --- internal/supervisor/collector_config.yaml.tmpl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/internal/supervisor/collector_config.yaml.tmpl b/internal/supervisor/collector_config.yaml.tmpl index 0e4c820..01c6bed 100644 --- a/internal/supervisor/collector_config.yaml.tmpl +++ b/internal/supervisor/collector_config.yaml.tmpl @@ -53,6 +53,23 @@ exporters: max_elapsed_time: 120s service: + telemetry: + metrics: + level: normal # Options: none, basic, normal, detailed + readers: + - periodic: + interval: 60s # How often to send internal metrics + exporter: + otlp: + # Directing internal metrics to your existing exporter + endpoint: {{ .ExporterEndpoint }} + protocol: grpc + headers: + - name: Host + value: insights-otlp.digitalocean.com + # Optional: Send collector internal logs as OTLP logs too + logs: + level: info pipelines: logs: receivers: [journald] From 9c066e153f5a28002ce8907eec4973cc77e50b4d Mon Sep 17 00:00:00 2001 From: Nitish Date: Wed, 13 May 2026 17:33:51 +0530 Subject: [PATCH 3/4] Added internal telemtry config changes --- internal/supervisor/collector_config.go | 10 ++++++++-- internal/supervisor/collector_config.yaml.tmpl | 7 ++++--- internal/supervisor/collector_config_test.go | 6 ++++++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/internal/supervisor/collector_config.go b/internal/supervisor/collector_config.go index 9dec53f..d6e6f75 100644 --- a/internal/supervisor/collector_config.go +++ b/internal/supervisor/collector_config.go @@ -17,11 +17,17 @@ var collectorConfigTmpl string var parsedCollectorConfigTmpl = template.Must(template.New("collector_config").Parse(collectorConfigTmpl)) +// insightsOTLPHost is the OTLP TLS server name. Internal telemetry gRPC uses it as the dial +// host so SNI matches the certificate; it must resolve to the VPC OTLP IP (not public DNS only). +const insightsOTLPHost = "insights-otlp.digitalocean.com" + func buildCollectorConfig(ip net.IP) ([]byte, error) { data := struct { - ExporterEndpoint string + ExporterEndpoint string + InternalOTLPEndpoint string }{ - ExporterEndpoint: net.JoinHostPort(ip.String(), "443"), + ExporterEndpoint: net.JoinHostPort(ip.String(), "443"), + InternalOTLPEndpoint: net.JoinHostPort(insightsOTLPHost, "443"), } var buf bytes.Buffer if err := parsedCollectorConfigTmpl.Execute(&buf, data); err != nil { diff --git a/internal/supervisor/collector_config.yaml.tmpl b/internal/supervisor/collector_config.yaml.tmpl index 01c6bed..d8ff699 100644 --- a/internal/supervisor/collector_config.yaml.tmpl +++ b/internal/supervisor/collector_config.yaml.tmpl @@ -58,11 +58,12 @@ service: level: normal # Options: none, basic, normal, detailed readers: - periodic: - interval: 60s # How often to send internal metrics + interval: 60000 # How often to send internal metrics exporter: otlp: - # Directing internal metrics to your existing exporter - endpoint: {{ .ExporterEndpoint }} + # Internal OTLP must use a hostname for TLS SNI (SDK path has no server_name_override). + # Must resolve to the same VPC target as ExporterEndpoint (DNS or /etc/hosts). + endpoint: {{ .InternalOTLPEndpoint }} protocol: grpc headers: - name: Host diff --git a/internal/supervisor/collector_config_test.go b/internal/supervisor/collector_config_test.go index 2954002..02733bf 100644 --- a/internal/supervisor/collector_config_test.go +++ b/internal/supervisor/collector_config_test.go @@ -40,6 +40,12 @@ func TestBuildConfig(t *testing.T) { if !strings.Contains(s, "server_name_override: insights-otlp.digitalocean.com") { t.Fatalf("server_name_override not found in config:\n%s", s) } + if !strings.Contains(s, "interval: 60000") { + t.Fatalf("internal telemetry periodic interval (ms) not found in config:\n%s", s) + } + if !strings.Contains(s, "endpoint: insights-otlp.digitalocean.com:443") { + t.Fatalf("internal telemetry OTLP hostname endpoint not found in config:\n%s", s) + } }) } } From 6c41ce9e213a7f72fb0bf391dacc1e44fd72270e Mon Sep 17 00:00:00 2001 From: Nitish Date: Thu, 14 May 2026 16:26:36 +0530 Subject: [PATCH 4/4] Added internal observability --- internal/supervisor/collector_config.go | 10 +---- .../supervisor/collector_config.yaml.tmpl | 43 +++++++++++++------ internal/supervisor/collector_config_test.go | 8 ++-- 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/internal/supervisor/collector_config.go b/internal/supervisor/collector_config.go index d6e6f75..9dec53f 100644 --- a/internal/supervisor/collector_config.go +++ b/internal/supervisor/collector_config.go @@ -17,17 +17,11 @@ var collectorConfigTmpl string var parsedCollectorConfigTmpl = template.Must(template.New("collector_config").Parse(collectorConfigTmpl)) -// insightsOTLPHost is the OTLP TLS server name. Internal telemetry gRPC uses it as the dial -// host so SNI matches the certificate; it must resolve to the VPC OTLP IP (not public DNS only). -const insightsOTLPHost = "insights-otlp.digitalocean.com" - func buildCollectorConfig(ip net.IP) ([]byte, error) { data := struct { - ExporterEndpoint string - InternalOTLPEndpoint string + ExporterEndpoint string }{ - ExporterEndpoint: net.JoinHostPort(ip.String(), "443"), - InternalOTLPEndpoint: net.JoinHostPort(insightsOTLPHost, "443"), + ExporterEndpoint: net.JoinHostPort(ip.String(), "443"), } var buf bytes.Buffer if err := parsedCollectorConfigTmpl.Execute(&buf, data); err != nil { diff --git a/internal/supervisor/collector_config.yaml.tmpl b/internal/supervisor/collector_config.yaml.tmpl index d8ff699..013ff0d 100644 --- a/internal/supervisor/collector_config.yaml.tmpl +++ b/internal/supervisor/collector_config.yaml.tmpl @@ -1,4 +1,12 @@ receivers: + prometheus/internal: + config: + scrape_configs: + - job_name: otelcol + scrape_interval: 60s + static_configs: + - targets: ['127.0.0.1:9464'] + journald: units: [] priority: info @@ -30,10 +38,22 @@ processors: # Hard limit = limit_mib = 180 MiB. limit_mib: 180 spike_limit_mib: 40 + transform/internal: + metric_statements: + - context: metric + statements: + - set(name, Concat(["do_", name], "")) + - context: datapoint + statements: + - set(attributes["do-obsd-ver"], "0.1") batch/logs: send_batch_size: 1024 send_batch_max_size: 2048 timeout: 5s + batch/metrics: + send_batch_size: 1024 + send_batch_max_size: 2048 + timeout: 5s exporters: otlp_grpc: @@ -55,24 +75,19 @@ exporters: service: telemetry: metrics: - level: normal # Options: none, basic, normal, detailed + level: normal readers: - - periodic: - interval: 60000 # How often to send internal metrics + - pull: exporter: - otlp: - # Internal OTLP must use a hostname for TLS SNI (SDK path has no server_name_override). - # Must resolve to the same VPC target as ExporterEndpoint (DNS or /etc/hosts). - endpoint: {{ .InternalOTLPEndpoint }} - protocol: grpc - headers: - - name: Host - value: insights-otlp.digitalocean.com - # Optional: Send collector internal logs as OTLP logs too - logs: - level: info + prometheus: + host: 127.0.0.1 + port: 9464 pipelines: logs: receivers: [journald] processors: [memory_limiter, batch/logs] exporters: [otlp_grpc] + metrics/internal: + receivers: [prometheus/internal] + processors: [memory_limiter, transform/internal, batch/metrics] + exporters: [otlp_grpc] diff --git a/internal/supervisor/collector_config_test.go b/internal/supervisor/collector_config_test.go index 02733bf..a70a0e0 100644 --- a/internal/supervisor/collector_config_test.go +++ b/internal/supervisor/collector_config_test.go @@ -40,11 +40,11 @@ func TestBuildConfig(t *testing.T) { if !strings.Contains(s, "server_name_override: insights-otlp.digitalocean.com") { t.Fatalf("server_name_override not found in config:\n%s", s) } - if !strings.Contains(s, "interval: 60000") { - t.Fatalf("internal telemetry periodic interval (ms) not found in config:\n%s", s) + if !strings.Contains(s, "level: normal") { + t.Fatalf("telemetry metrics level: normal not found in config:\n%s", s) } - if !strings.Contains(s, "endpoint: insights-otlp.digitalocean.com:443") { - t.Fatalf("internal telemetry OTLP hostname endpoint not found in config:\n%s", s) + if !strings.Contains(s, "scrape_interval: 60s") { + t.Fatalf("scrape_interval: 60s not found in config:\n%s", s) } }) }