From 8f7f60f4c763c7e574b9e0af54368a6a14d299b3 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Mon, 23 Mar 2026 09:30:31 -0400 Subject: [PATCH 01/58] Allow Config to resolve traceURL; pass this as a field into httptransport --- ddtrace/tracer/option.go | 27 ++++--- ddtrace/tracer/tracer_test.go | 4 +- ddtrace/tracer/transport.go | 19 +++-- ddtrace/tracer/transport_bench_test.go | 4 +- ddtrace/tracer/transport_test.go | 10 +-- internal/config/config.go | 16 +++++ internal/config/config_helpers.go | 72 +++++++++++++++++-- .../config/provider/otelenvconfigsource.go | 23 +++++- 8 files changed, 137 insertions(+), 38 deletions(-) diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 2ec61e66093..da1b30549e2 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -134,8 +134,9 @@ var ( // Supported trace protocols. const ( - traceProtocolV04 = internalconfig.TraceProtocolV04 - traceProtocolV1 = internalconfig.TraceProtocolV1 + traceProtocolV04 = internalconfig.TraceProtocolV04 + traceProtocolV1 = internalconfig.TraceProtocolV1 + traceProtocolOTLP = internalconfig.TraceProtocolOTLP ) // config holds the tracer configuration. @@ -426,7 +427,8 @@ func newConfig(opts ...StartOption) (*config, error) { } } if c.transport == nil { - c.transport = newHTTPTransport(c.internalConfig.AgentURL().String(), c.httpClient) + agentURL := c.internalConfig.AgentURL().String() + c.transport = newHTTPTransport(c.internalConfig.TraceURL(), agentURL, c.httpClient) } if c.propagator == nil { envKey := "DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH" @@ -465,13 +467,8 @@ func newConfig(opts ...StartOption) (*config, error) { af := loadAgentFeatures(agentDisabled, agentURL, c.httpClient) c.agent.store(af) // If the agent doesn't support the v1 protocol, downgrade to v0.4 - if !af.v1ProtocolAvailable { - c.internalConfig.SetTraceProtocol(traceProtocolV04, internalconfig.OriginCalculated) - } - if c.internalConfig.TraceProtocol() == traceProtocolV1 { - if t, ok := c.transport.(*httpTransport); ok { - t.traceURL = fmt.Sprintf("%s%s", agentURL.String(), tracesAPIPathV1) - } + if c.internalConfig.TraceProtocol() == traceProtocolV1 && !af.v1ProtocolAvailable { + c.setTraceEndpoint(traceProtocolV04, fmt.Sprintf("%s%s", agentURL.String(), tracesAPIPath)) } info, ok := debug.ReadBuildInfo() @@ -794,6 +791,16 @@ func loadAgentFeatures(agentDisabled bool, agentURL *url.URL, httpClient *http.C return features } +// setTraceEndpoint updates the trace protocol, trace URL on the config, and +// the traceURL on the active transport (if it is an httpTransport) in one step. +func (c *config) setTraceEndpoint(protocol float64, traceURL string) { + c.internalConfig.SetTraceProtocol(protocol, internalconfig.OriginCalculated) + c.internalConfig.SetTraceURL(traceURL, internalconfig.OriginCalculated) + if t, ok := c.transport.(*httpTransport); ok { + t.traceURL = traceURL + } +} + // agentEnabled reports whether the tracer should communicate with the agent. // The agent is considered disabled in serverless (LogToStdout), when the // tracer itself is disabled, or in CI visibility agentless mode. diff --git a/ddtrace/tracer/tracer_test.go b/ddtrace/tracer/tracer_test.go index d708fffe7fd..6b3936f0e0f 100644 --- a/ddtrace/tracer/tracer_test.go +++ b/ddtrace/tracer/tracer_test.go @@ -1239,7 +1239,7 @@ func TestTracerNoDebugStack(t *testing.T) { // newDefaultTransport return a default transport for this tracing client func newDefaultTransport() transport { - return newHTTPTransport(defaultURL, internal.DefaultHTTPClient(defaultHTTPTimeout, true)) + return newHTTPTransport(defaultURL+tracesAPIPath, defaultURL, internal.DefaultHTTPClient(defaultHTTPTimeout, true)) } func TestNewSpan(t *testing.T) { @@ -1358,7 +1358,7 @@ func TestTracerPrioritySampler(t *testing.T) { url := "http://" + srv.Listener.Addr().String() tr, _, flush, stop, err := startTestTracer(t, - withTransport(newHTTPTransport(url, internal.DefaultHTTPClient(defaultHTTPTimeout, false))), + withTransport(newHTTPTransport(url+tracesAPIPath, url, internal.DefaultHTTPClient(defaultHTTPTimeout, false))), ) assert.Nil(err) defer stop() diff --git a/ddtrace/tracer/transport.go b/ddtrace/tracer/transport.go index 8c58eb86d19..4f3e583942f 100644 --- a/ddtrace/tracer/transport.go +++ b/ddtrace/tracer/transport.go @@ -63,15 +63,12 @@ type httpTransport struct { headers map[string]string // the Transport headers } -// newTransport returns a new Transport implementation that sends traces to a -// trace agent at the given url, using a given *http.Client. -// -// In general, using this method is only necessary if you have a trace agent -// running on a non-default port, if it's located on another machine, or when -// otherwise needing to customize the transport layer, for instance when using -// a unix domain socket. -func newHTTPTransport(url string, client *http.Client) *httpTransport { - // initialize the default EncoderPool with Encoder headers +// newHTTPTransport returns a new Transport implementation that sends traces +// to the given traceURL and stats to the given agentURL, using the provided +// *http.Client. The traceURL should be the fully resolved endpoint (including +// path); the agentURL is the base agent address used to derive the stats +// endpoint. +func newHTTPTransport(traceURL string, agentURL string, client *http.Client) *httpTransport { defaultHeaders := map[string]string{ "Datadog-Meta-Lang": "go", "Datadog-Meta-Lang-Version": strings.TrimPrefix(runtime.Version(), "go"), @@ -89,8 +86,8 @@ func newHTTPTransport(url string, client *http.Client) *httpTransport { defaultHeaders["Datadog-External-Env"] = extEnv } return &httpTransport{ - traceURL: fmt.Sprintf("%s%s", url, tracesAPIPath), - statsURL: fmt.Sprintf("%s%s", url, statsAPIPath), + traceURL: traceURL, + statsURL: fmt.Sprintf("%s%s", agentURL, statsAPIPath), client: client, headers: defaultHeaders, } diff --git a/ddtrace/tracer/transport_bench_test.go b/ddtrace/tracer/transport_bench_test.go index 163f79b5a8d..f3f3306bb1e 100644 --- a/ddtrace/tracer/transport_bench_test.go +++ b/ddtrace/tracer/transport_bench_test.go @@ -24,7 +24,7 @@ func BenchmarkHTTPTransportSend(b *testing.B) { })) defer server.Close() - transport := newHTTPTransport(server.URL, internal.DefaultHTTPClient(5*time.Second, false)) + transport := newHTTPTransport(server.URL+tracesAPIPath, server.URL, internal.DefaultHTTPClient(5*time.Second, false)) payloadSizes := []struct { name string @@ -69,7 +69,7 @@ func BenchmarkTransportSendConcurrent(b *testing.B) { })) defer server.Close() - transport := newHTTPTransport(server.URL, internal.DefaultHTTPClient(5*time.Second, false)) + transport := newHTTPTransport(server.URL+tracesAPIPath, server.URL, internal.DefaultHTTPClient(5*time.Second, false)) concurrencyLevels := []int{1, 2, 4, 8} for _, concurrency := range concurrencyLevels { diff --git a/ddtrace/tracer/transport_test.go b/ddtrace/tracer/transport_test.go index 4e6d491cb4f..7c5548ab222 100644 --- a/ddtrace/tracer/transport_test.go +++ b/ddtrace/tracer/transport_test.go @@ -90,7 +90,7 @@ func TestTracesAgentIntegration(t *testing.T) { } for _, tc := range testCases { - transport := newHTTPTransport(defaultURL, internal.DefaultHTTPClient(defaultHTTPTimeout, false)) + transport := newHTTPTransport(defaultURL+tracesAPIPath, defaultURL, internal.DefaultHTTPClient(defaultHTTPTimeout, false)) p, err := encode(tc.payload) assert.NoError(err) body, err := transport.send(p) @@ -167,7 +167,7 @@ func TestTransportResponse(t *testing.T) { w.Write([]byte(tt.body)) })) defer srv.Close() - transport := newHTTPTransport(srv.URL, internal.DefaultHTTPClient(defaultHTTPTimeout, false)) + transport := newHTTPTransport(srv.URL+tracesAPIPath, srv.URL, internal.DefaultHTTPClient(defaultHTTPTimeout, false)) rc, err := transport.send(newPayload(traceProtocolV04)) if tt.err != "" { assert.Equal(tt.err, err.Error()) @@ -207,7 +207,7 @@ func TestTraceCountHeader(t *testing.T) { })) defer srv.Close() for _, tc := range testCases { - transport := newHTTPTransport(srv.URL, internal.DefaultHTTPClient(defaultHTTPTimeout, false)) + transport := newHTTPTransport(srv.URL+tracesAPIPath, srv.URL, internal.DefaultHTTPClient(defaultHTTPTimeout, false)) p, err := encode(tc.payload) assert.NoError(err) _, err = transport.send(p) @@ -249,7 +249,7 @@ func TestCustomTransport(t *testing.T) { c := &http.Client{} crt := wrapRecordingRoundTripper(c) - transport := newHTTPTransport(srv.URL, c) + transport := newHTTPTransport(srv.URL+tracesAPIPath, srv.URL, c) p, err := encode(getTestTrace(1, 1)) assert.NoError(err) _, err = transport.send(p) @@ -634,7 +634,7 @@ func TestConcurrentTraceFlushOverUDS(t *testing.T) { udsURL := internal.UnixDataSocketURL(socketPath).String() client := internal.UDSClient(socketPath, 5*time.Second) - transport := newHTTPTransport(udsURL, client) + transport := newHTTPTransport(udsURL+tracesAPIPath, udsURL, client) const numGoroutines = 20 diff --git a/internal/config/config.go b/internal/config/config.go index be49aab6105..e55b60bf3ba 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -111,6 +111,9 @@ type Config struct { // Initialized from DD_TRACE_AGENT_PROTOCOL_VERSION, may be downgraded by the tracer // if the agent doesn't support the requested version. traceProtocol float64 + // traceURL is the endpoint URL for sending traces, resolved from + // the active traceProtocol, agent URL, and OTLP env vars. + traceURL string } // loadConfig initializes and returns a new config by reading from all configured sources. @@ -155,6 +158,7 @@ func loadConfig() *Config { cfg.retryInterval = p.GetDuration("DD_TRACE_RETRY_INTERVAL", time.Millisecond) cfg.logsOTelEnabled = p.GetBool("DD_LOGS_OTEL_ENABLED", false) cfg.traceProtocol = resolveTraceProtocol(p.GetStringWithValidator("DD_TRACE_AGENT_PROTOCOL_VERSION", "0.4", validateTraceProtocolVersion)) + cfg.traceURL = resolveTraceURL(cfg.traceProtocol, cfg.agentURL) // Parse feature flags from DD_TRACE_FEATURES as a set cfg.featureFlags = make(map[string]struct{}) @@ -701,3 +705,15 @@ func (c *Config) SetTraceProtocol(v float64, origin telemetry.Origin) { c.traceProtocol = v configtelemetry.Report("DD_TRACE_AGENT_PROTOCOL_VERSION", v, origin) } + +func (c *Config) TraceURL() string { + c.mu.RLock() + defer c.mu.RUnlock() + return c.traceURL +} + +func (c *Config) SetTraceURL(url string, origin telemetry.Origin) { + c.mu.Lock() + defer c.mu.Unlock() + c.traceURL = url +} diff --git a/internal/config/config_helpers.go b/internal/config/config_helpers.go index 1ac24e3471a..3988c8ee97a 100644 --- a/internal/config/config_helpers.go +++ b/internal/config/config_helpers.go @@ -6,11 +6,14 @@ package config import ( + "fmt" "net" "net/url" "os" + "strings" "github.com/DataDog/dd-trace-go/v2/internal" + "github.com/DataDog/dd-trace-go/v2/internal/env" "github.com/DataDog/dd-trace-go/v2/internal/log" ) @@ -26,15 +29,25 @@ const ( TraceMaxSize = int(1e5) // Trace protocol versions (agent wire format). - TraceProtocolV04 = 0.4 // default - TraceProtocolV1 = 1.0 - TraceProtocolVersionStringV04 = "0.4" - TraceProtocolVersionStringV1 = "1.0" + TraceProtocolV04 = 0.4 // default + TraceProtocolV1 = 1.0 + TraceProtocolOTLP = 2.0 + TraceProtocolVersionStringV04 = "0.4" + TraceProtocolVersionStringV1 = "1.0" + TraceProtocolVersionStringOTLP = "otlp" // Agent URL schemes supported by DD_TRACE_AGENT_URL. URLSchemeUnix = "unix" URLSchemeHTTP = "http" URLSchemeHTTPS = "https" + + // Trace API paths appended to the agent URL for each protocol. + TracesPathV04 = "/v0.4/traces" + TracesPathV1 = "/v1.0/traces" + + // OTLP standard traces path and default collector port. + otlpTracesPath = "/v1/traces" + otlpDefaultPort = "4318" ) func validateSampleRate(rate float64) bool { @@ -70,10 +83,16 @@ func validateTraceProtocolVersion(v string) bool { } func resolveTraceProtocol(v string) float64 { - if v == TraceProtocolVersionStringV1 { + switch v { + case TraceProtocolVersionStringV04: + return TraceProtocolV04 + case TraceProtocolVersionStringV1: return TraceProtocolV1 + case TraceProtocolVersionStringOTLP: + return TraceProtocolOTLP + default: + return TraceProtocolV04 } - return TraceProtocolV04 } // resolveAgentURL computes the final agent URL from the three env-var strings @@ -130,3 +149,44 @@ func detectUDSURL() *url.URL { Path: internal.DefaultTraceAgentUDSPath, } } + +// resolveTraceURL computes the full trace endpoint URL based on the protocol +// and agent URL. For Datadog protocols the URL is derived from agentURL; for +// OTLP it follows the standard env-var fallback chain. +func resolveTraceURL(protocol float64, rawAgentURL *url.URL) string { + agentHTTPURL := rawAgentURL + if rawAgentURL != nil && rawAgentURL.Scheme == URLSchemeUnix { + agentHTTPURL = internal.UnixDataSocketURL(rawAgentURL.Path) + } + + switch protocol { + case TraceProtocolOTLP: + return resolveOTLPTraceURL(rawAgentURL) + case TraceProtocolV1: + return agentHTTPURL.String() + TracesPathV1 + default: + return agentHTTPURL.String() + TracesPathV04 + } +} + +// resolveOTLPTraceURL resolves the OTLP trace endpoint using the standard +// OTel env-var priority: +// 1. OTEL_EXPORTER_OTLP_TRACES_ENDPOINT (full URL) +// 2. OTEL_EXPORTER_OTLP_ENDPOINT (base URL) + /v1/traces +// 3. agentURL host + default OTLP port 4318 + /v1/traces +func resolveOTLPTraceURL(rawAgentURL *url.URL) string { + // TODO: Potentially check these env vars in configprovider instead, if they need to be supported across config sources and report telemetry + if v := env.Get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"); v != "" { + return v + } + if v := env.Get("OTEL_EXPORTER_OTLP_ENDPOINT"); v != "" { + return strings.TrimRight(v, "/") + otlpTracesPath + } + host := internal.DefaultAgentHostname + if rawAgentURL != nil { + if h := rawAgentURL.Hostname(); h != "" { + host = h + } + } + return fmt.Sprintf("http://%s:%s%s", host, otlpDefaultPort, otlpTracesPath) +} diff --git a/internal/config/provider/otelenvconfigsource.go b/internal/config/provider/otelenvconfigsource.go index be001d7fb9c..54a209aa95b 100644 --- a/internal/config/provider/otelenvconfigsource.go +++ b/internal/config/provider/otelenvconfigsource.go @@ -85,6 +85,10 @@ var otelConfigs = map[string]*otelDDEnv{ ot: "OTEL_RESOURCE_ATTRIBUTES", remapper: mapDDTags, }, + "DD_TRACE_AGENT_PROTOCOL_VERSION": { + ot: "OTEL_TRACES_EXPORTER", + remapper: mapTraceProtocol, + }, } var ddTagsMapping = map[string]string{ @@ -131,10 +135,14 @@ func mapLogLevel(ot string) (string, error) { // mapEnabled maps OTEL_TRACES_EXPORTER to DD_TRACE_ENABLED func mapEnabled(ot string) (string, error) { - if strings.TrimSpace(strings.ToLower(ot)) == "none" { + switch strings.TrimSpace(strings.ToLower(ot)) { + case "none": return "false", nil + case "otlp": + return "true", nil // Handled separately by mapTraceProtocol + default: + return "", fmt.Errorf("the following configuration is not supported: OTEL_TRACES_EXPORTER=%v", ot) } - return "", fmt.Errorf("the following configuration is not supported: OTEL_TRACES_EXPORTER=%v", ot) } // otelTraceIDRatio returns the value of OTEL_TRACES_SAMPLER_ARG if set, otherwise "1.0" @@ -198,3 +206,14 @@ func mapDDTags(ot string) (string, error) { return strings.Join(ddTags, ","), nil } + +func mapTraceProtocol(ot string) (string, error) { + switch strings.TrimSpace(strings.ToLower(ot)) { + case "none": + return "", nil // not relevant to protocol + case "otlp": + return "2.0", nil + default: + return "", fmt.Errorf("the following configuration is not supported: OTEL_TRACES_EXPORTER=%v", ot) + } +} From 3e3fec8c9165976d20f7cbec81130dfc0f2fbcec Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Mon, 23 Mar 2026 10:24:32 -0400 Subject: [PATCH 02/58] Fix otlp protocol resolution and add tests for all new logic --- ddtrace/tracer/option.go | 15 +- internal/config/config.go | 11 +- internal/config/config_helpers.go | 20 ++- internal/config/config_helpers_test.go | 84 ++++++++++++ internal/config/config_test.go | 128 ++++++++++++++++++ .../config/provider/otelenvconfigsource.go | 2 +- .../provider/otelenvconfigsource_test.go | 21 +++ internal/env/supported_configurations.gen.go | 1 + internal/env/supported_configurations.json | 21 ++- 9 files changed, 266 insertions(+), 37 deletions(-) create mode 100644 internal/config/config_helpers_test.go diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index da1b30549e2..10932e14391 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -468,7 +468,10 @@ func newConfig(opts ...StartOption) (*config, error) { c.agent.store(af) // If the agent doesn't support the v1 protocol, downgrade to v0.4 if c.internalConfig.TraceProtocol() == traceProtocolV1 && !af.v1ProtocolAvailable { - c.setTraceEndpoint(traceProtocolV04, fmt.Sprintf("%s%s", agentURL.String(), tracesAPIPath)) + c.internalConfig.SetTraceProtocol(traceProtocolV04, internalconfig.OriginCalculated) + if t, ok := c.transport.(*httpTransport); ok { + t.traceURL = c.internalConfig.TraceURL() + } } info, ok := debug.ReadBuildInfo() @@ -791,16 +794,6 @@ func loadAgentFeatures(agentDisabled bool, agentURL *url.URL, httpClient *http.C return features } -// setTraceEndpoint updates the trace protocol, trace URL on the config, and -// the traceURL on the active transport (if it is an httpTransport) in one step. -func (c *config) setTraceEndpoint(protocol float64, traceURL string) { - c.internalConfig.SetTraceProtocol(protocol, internalconfig.OriginCalculated) - c.internalConfig.SetTraceURL(traceURL, internalconfig.OriginCalculated) - if t, ok := c.transport.(*httpTransport); ok { - t.traceURL = traceURL - } -} - // agentEnabled reports whether the tracer should communicate with the agent. // The agent is considered disabled in serverless (LogToStdout), when the // tracer itself is disabled, or in CI visibility agentless mode. diff --git a/internal/config/config.go b/internal/config/config.go index e55b60bf3ba..0ca2b49e87b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -158,7 +158,9 @@ func loadConfig() *Config { cfg.retryInterval = p.GetDuration("DD_TRACE_RETRY_INTERVAL", time.Millisecond) cfg.logsOTelEnabled = p.GetBool("DD_LOGS_OTEL_ENABLED", false) cfg.traceProtocol = resolveTraceProtocol(p.GetStringWithValidator("DD_TRACE_AGENT_PROTOCOL_VERSION", "0.4", validateTraceProtocolVersion)) - cfg.traceURL = resolveTraceURL(cfg.traceProtocol, cfg.agentURL) + otlpTracesEndpoint := p.GetString("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") + otlpEndpoint := p.GetString("OTEL_EXPORTER_OTLP_ENDPOINT", "") + cfg.traceURL = resolveTraceURL(cfg.traceProtocol, cfg.agentURL, otlpTracesEndpoint, otlpEndpoint) // Parse feature flags from DD_TRACE_FEATURES as a set cfg.featureFlags = make(map[string]struct{}) @@ -703,6 +705,7 @@ func (c *Config) SetTraceProtocol(v float64, origin telemetry.Origin) { c.mu.Lock() defer c.mu.Unlock() c.traceProtocol = v + c.traceURL = resolveTraceURL(v, c.agentURL, "", "") configtelemetry.Report("DD_TRACE_AGENT_PROTOCOL_VERSION", v, origin) } @@ -711,9 +714,3 @@ func (c *Config) TraceURL() string { defer c.mu.RUnlock() return c.traceURL } - -func (c *Config) SetTraceURL(url string, origin telemetry.Origin) { - c.mu.Lock() - defer c.mu.Unlock() - c.traceURL = url -} diff --git a/internal/config/config_helpers.go b/internal/config/config_helpers.go index 3988c8ee97a..6e1345bf74d 100644 --- a/internal/config/config_helpers.go +++ b/internal/config/config_helpers.go @@ -13,7 +13,6 @@ import ( "strings" "github.com/DataDog/dd-trace-go/v2/internal" - "github.com/DataDog/dd-trace-go/v2/internal/env" "github.com/DataDog/dd-trace-go/v2/internal/log" ) @@ -79,7 +78,7 @@ func validatePartialFlushMinSpans(minSpans int) bool { } func validateTraceProtocolVersion(v string) bool { - return v == TraceProtocolVersionStringV04 || v == TraceProtocolVersionStringV1 + return v == TraceProtocolVersionStringV04 || v == TraceProtocolVersionStringV1 || v == TraceProtocolVersionStringOTLP } func resolveTraceProtocol(v string) float64 { @@ -152,8 +151,8 @@ func detectUDSURL() *url.URL { // resolveTraceURL computes the full trace endpoint URL based on the protocol // and agent URL. For Datadog protocols the URL is derived from agentURL; for -// OTLP it follows the standard env-var fallback chain. -func resolveTraceURL(protocol float64, rawAgentURL *url.URL) string { +// OTLP it uses the provided endpoint values (sourced from the config provider). +func resolveTraceURL(protocol float64, rawAgentURL *url.URL, otlpTracesEndpoint, otlpEndpoint string) string { agentHTTPURL := rawAgentURL if rawAgentURL != nil && rawAgentURL.Scheme == URLSchemeUnix { agentHTTPURL = internal.UnixDataSocketURL(rawAgentURL.Path) @@ -161,7 +160,7 @@ func resolveTraceURL(protocol float64, rawAgentURL *url.URL) string { switch protocol { case TraceProtocolOTLP: - return resolveOTLPTraceURL(rawAgentURL) + return resolveOTLPTraceURL(rawAgentURL, otlpTracesEndpoint, otlpEndpoint) case TraceProtocolV1: return agentHTTPURL.String() + TracesPathV1 default: @@ -174,13 +173,12 @@ func resolveTraceURL(protocol float64, rawAgentURL *url.URL) string { // 1. OTEL_EXPORTER_OTLP_TRACES_ENDPOINT (full URL) // 2. OTEL_EXPORTER_OTLP_ENDPOINT (base URL) + /v1/traces // 3. agentURL host + default OTLP port 4318 + /v1/traces -func resolveOTLPTraceURL(rawAgentURL *url.URL) string { - // TODO: Potentially check these env vars in configprovider instead, if they need to be supported across config sources and report telemetry - if v := env.Get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"); v != "" { - return v +func resolveOTLPTraceURL(rawAgentURL *url.URL, otlpTracesEndpoint, otlpEndpoint string) string { + if otlpTracesEndpoint != "" { + return otlpTracesEndpoint } - if v := env.Get("OTEL_EXPORTER_OTLP_ENDPOINT"); v != "" { - return strings.TrimRight(v, "/") + otlpTracesPath + if otlpEndpoint != "" { + return strings.TrimRight(otlpEndpoint, "/") + otlpTracesPath } host := internal.DefaultAgentHostname if rawAgentURL != nil { diff --git a/internal/config/config_helpers_test.go b/internal/config/config_helpers_test.go new file mode 100644 index 00000000000..4b706433e5b --- /dev/null +++ b/internal/config/config_helpers_test.go @@ -0,0 +1,84 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2025 Datadog, Inc. + +package config + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestResolveTraceURL(t *testing.T) { + httpAgent := &url.URL{Scheme: "http", Host: "myhost:8126"} + unixAgent := &url.URL{Scheme: "unix", Path: "/var/run/datadog/apm.socket"} + + t.Run("v0.4 protocol uses agent URL", func(t *testing.T) { + got := resolveTraceURL(TraceProtocolV04, httpAgent, "", "") + assert.Equal(t, "http://myhost:8126/v0.4/traces", got) + }) + + t.Run("v1 protocol uses agent URL", func(t *testing.T) { + got := resolveTraceURL(TraceProtocolV1, httpAgent, "", "") + assert.Equal(t, "http://myhost:8126/v1.0/traces", got) + }) + + t.Run("v0.4 with unix socket rewrites to HTTP", func(t *testing.T) { + got := resolveTraceURL(TraceProtocolV04, unixAgent, "", "") + assert.Contains(t, got, "/v0.4/traces") + assert.Contains(t, got, "http://") + }) + + t.Run("OTLP delegates to resolveOTLPTraceURL", func(t *testing.T) { + got := resolveTraceURL(TraceProtocolOTLP, httpAgent, "", "") + assert.Equal(t, "http://myhost:4318/v1/traces", got) + }) + + t.Run("OTLP with traces endpoint set", func(t *testing.T) { + got := resolveTraceURL(TraceProtocolOTLP, httpAgent, "http://collector:4318/v1/traces", "") + assert.Equal(t, "http://collector:4318/v1/traces", got) + }) + + t.Run("OTLP with base endpoint set", func(t *testing.T) { + got := resolveTraceURL(TraceProtocolOTLP, httpAgent, "", "http://collector:4318") + assert.Equal(t, "http://collector:4318/v1/traces", got) + }) +} + +func TestResolveOTLPTraceURL(t *testing.T) { + httpAgent := &url.URL{Scheme: "http", Host: "myhost:8126"} + + t.Run("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT takes priority", func(t *testing.T) { + got := resolveOTLPTraceURL(httpAgent, "http://traces-collector:4318/v1/traces", "http://base-collector:4318") + assert.Equal(t, "http://traces-collector:4318/v1/traces", got) + }) + + t.Run("OTEL_EXPORTER_OTLP_ENDPOINT used as base with path appended", func(t *testing.T) { + got := resolveOTLPTraceURL(httpAgent, "", "http://collector:4318") + assert.Equal(t, "http://collector:4318/v1/traces", got) + }) + + t.Run("OTEL_EXPORTER_OTLP_ENDPOINT trailing slash is trimmed", func(t *testing.T) { + got := resolveOTLPTraceURL(httpAgent, "", "http://collector:4318/") + assert.Equal(t, "http://collector:4318/v1/traces", got) + }) + + t.Run("default uses agent host with OTLP port", func(t *testing.T) { + got := resolveOTLPTraceURL(httpAgent, "", "") + assert.Equal(t, "http://myhost:4318/v1/traces", got) + }) + + t.Run("default with nil agent URL uses localhost", func(t *testing.T) { + got := resolveOTLPTraceURL(nil, "", "") + assert.Equal(t, "http://localhost:4318/v1/traces", got) + }) + + t.Run("default with unix socket agent uses localhost", func(t *testing.T) { + unixAgent := &url.URL{Scheme: "unix", Path: "/var/run/datadog/apm.socket"} + got := resolveOTLPTraceURL(unixAgent, "", "") + assert.Equal(t, "http://localhost:4318/v1/traces", got) + }) +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 06947473816..cca9be8ca43 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -461,6 +461,134 @@ func TestSetServiceMappingReportsFullList(t *testing.T) { assert.Equal(t, []string{"a:3", "b:2"}, parts) } +func TestTraceURLResolution(t *testing.T) { + t.Run("default protocol resolves traceURL from agent URL", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + cfg := Get() + require.NotNil(t, cfg) + + traceURL := cfg.TraceURL() + assert.Contains(t, traceURL, "/v0.4/traces") + }) + + t.Run("OTLP protocol resolves traceURL with default OTLP port", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + + cfg := Get() + require.NotNil(t, cfg) + + assert.Equal(t, TraceProtocolOTLP, cfg.TraceProtocol()) + assert.Contains(t, cfg.TraceURL(), ":4318/v1/traces") + }) + + t.Run("OTLP with OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "http://collector:4318/v1/traces") + + cfg := Get() + require.NotNil(t, cfg) + + assert.Equal(t, "http://collector:4318/v1/traces", cfg.TraceURL()) + }) + + t.Run("OTLP with OTEL_EXPORTER_OTLP_ENDPOINT appends path", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4318") + + cfg := Get() + require.NotNil(t, cfg) + + assert.Equal(t, "http://collector:4318/v1/traces", cfg.TraceURL()) + }) + + t.Run("OTLP TRACES_ENDPOINT takes priority over ENDPOINT", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "http://specific:4318/v1/traces") + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://generic:4318") + + cfg := Get() + require.NotNil(t, cfg) + + assert.Equal(t, "http://specific:4318/v1/traces", cfg.TraceURL()) + }) + + t.Run("OTLP uses agent host when no OTLP endpoint configured", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + t.Setenv("DD_AGENT_HOST", "custom-agent") + + cfg := Get() + require.NotNil(t, cfg) + + assert.Equal(t, "http://custom-agent:4318/v1/traces", cfg.TraceURL()) + }) +} + +func TestDDTraceProtocolTakesPrecedenceOverOTEL(t *testing.T) { + t.Run("DD_TRACE_AGENT_PROTOCOL_VERSION overrides OTEL_TRACES_EXPORTER=otlp", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + t.Setenv("DD_TRACE_AGENT_PROTOCOL_VERSION", "0.4") + + cfg := Get() + require.NotNil(t, cfg) + + assert.Equal(t, TraceProtocolV04, cfg.TraceProtocol()) + assert.Contains(t, cfg.TraceURL(), "/v0.4/traces") + }) + + t.Run("DD_TRACE_AGENT_PROTOCOL_VERSION=1.0 overrides OTEL_TRACES_EXPORTER=otlp", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + t.Setenv("DD_TRACE_AGENT_PROTOCOL_VERSION", "1.0") + + cfg := Get() + require.NotNil(t, cfg) + + assert.Equal(t, TraceProtocolV1, cfg.TraceProtocol()) + assert.Contains(t, cfg.TraceURL(), "/v1.0/traces") + }) +} + +func TestSetTraceProtocolRecomputesTraceURL(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + cfg := Get() + require.NotNil(t, cfg) + + // Default should be v0.4 + assert.Contains(t, cfg.TraceURL(), "/v0.4/traces") + + // Switch to v1 + cfg.SetTraceProtocol(TraceProtocolV1, telemetry.OriginCode) + assert.Contains(t, cfg.TraceURL(), "/v1.0/traces") + + // Switch back to v0.4 + cfg.SetTraceProtocol(TraceProtocolV04, telemetry.OriginCode) + assert.Contains(t, cfg.TraceURL(), "/v0.4/traces") +} + func TestHostnameConfiguration(t *testing.T) { t.Run("default behavior - hostname empty when not configured", func(t *testing.T) { resetGlobalState() diff --git a/internal/config/provider/otelenvconfigsource.go b/internal/config/provider/otelenvconfigsource.go index 54a209aa95b..d95c7565441 100644 --- a/internal/config/provider/otelenvconfigsource.go +++ b/internal/config/provider/otelenvconfigsource.go @@ -212,7 +212,7 @@ func mapTraceProtocol(ot string) (string, error) { case "none": return "", nil // not relevant to protocol case "otlp": - return "2.0", nil + return "otlp", nil default: return "", fmt.Errorf("the following configuration is not supported: OTEL_TRACES_EXPORTER=%v", ot) } diff --git a/internal/config/provider/otelenvconfigsource_test.go b/internal/config/provider/otelenvconfigsource_test.go index c97c45d4a9f..2931b6fe98a 100644 --- a/internal/config/provider/otelenvconfigsource_test.go +++ b/internal/config/provider/otelenvconfigsource_test.go @@ -77,6 +77,27 @@ func TestOtelEnvConfigSource(t *testing.T) { assert.Equal(t, "false", v) }) + t.Run("maps OTEL_TRACES_EXPORTER=otlp to DD_TRACE_ENABLED=true", func(t *testing.T) { + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + source := &otelEnvConfigSource{} + v := source.get("DD_TRACE_ENABLED") + assert.Equal(t, "true", v) + }) + + t.Run("maps OTEL_TRACES_EXPORTER=otlp to DD_TRACE_AGENT_PROTOCOL_VERSION=otlp", func(t *testing.T) { + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + source := &otelEnvConfigSource{} + v := source.get("DD_TRACE_AGENT_PROTOCOL_VERSION") + assert.Equal(t, "otlp", v) + }) + + t.Run("OTEL_TRACES_EXPORTER=none does not set protocol", func(t *testing.T) { + t.Setenv("OTEL_TRACES_EXPORTER", "none") + source := &otelEnvConfigSource{} + v := source.get("DD_TRACE_AGENT_PROTOCOL_VERSION") + assert.Equal(t, "", v) + }) + t.Run("returns empty for invalid OTEL_TRACES_EXPORTER", func(t *testing.T) { telemetryClient := new(telemetrytest.RecordClient) defer telemetry.MockClient(telemetryClient)() diff --git a/internal/env/supported_configurations.gen.go b/internal/env/supported_configurations.gen.go index 0a6322e9ebe..9bc8ba83d7a 100644 --- a/internal/env/supported_configurations.gen.go +++ b/internal/env/supported_configurations.gen.go @@ -246,6 +246,7 @@ var SupportedConfigurations = map[string]struct{}{ "OTEL_EXPORTER_OTLP_METRICS_TIMEOUT": {}, "OTEL_EXPORTER_OTLP_PROTOCOL": {}, "OTEL_EXPORTER_OTLP_TIMEOUT": {}, + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": {}, "OTEL_LOGS_EXPORTER": {}, "OTEL_LOG_LEVEL": {}, "OTEL_METRICS_EXPORTER": {}, diff --git a/internal/env/supported_configurations.json b/internal/env/supported_configurations.json index 4cfdb7a0d12..c6e2d5bc737 100644 --- a/internal/env/supported_configurations.json +++ b/internal/env/supported_configurations.json @@ -910,6 +910,13 @@ "default": "8126" } ], + "DD_TRACE_AGENT_PROTOCOL_VERSION": [ + { + "implementation": "B", + "type": "string", + "default": "1.0" + } + ], "DD_TRACE_AGENT_URL": [ { "implementation": "C", @@ -1491,13 +1498,6 @@ "default": "false" } ], - "DD_TRACE_AGENT_PROTOCOL_VERSION": [ - { - "implementation": "B", - "type": "string", - "default": "1.0" - } - ], "DD_TRACE_VALKEY_ANALYTICS_ENABLED": [ { "implementation": "B", @@ -1666,6 +1666,13 @@ "default": "10000" } ], + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": [ + { + "implementation": "B", + "type": "string", + "default": "" + } + ], "OTEL_LOGS_EXPORTER": [ { "implementation": "A", From c559f8f2e7b2d9d2b469d4065e6b8db3e054d15b Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Mon, 23 Mar 2026 10:43:13 -0400 Subject: [PATCH 03/58] Implement OTEL_EXPORTER_OTLP_TRACES_HEADERS --- ddtrace/tracer/option.go | 6 +- ddtrace/tracer/tracer_test.go | 4 +- ddtrace/tracer/transport.go | 32 +++++---- ddtrace/tracer/transport_bench_test.go | 4 +- ddtrace/tracer/transport_test.go | 10 +-- internal/config/config.go | 15 +++++ internal/config/config_helpers.go | 38 +++++++++++ internal/config/config_helpers_test.go | 57 ++++++++++++++++ internal/config/config_test.go | 71 ++++++++++++++++++++ internal/env/supported_configurations.gen.go | 1 + internal/env/supported_configurations.json | 7 ++ 11 files changed, 221 insertions(+), 24 deletions(-) diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 10932e14391..3fa34b1876b 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -428,7 +428,11 @@ func newConfig(opts ...StartOption) (*config, error) { } if c.transport == nil { agentURL := c.internalConfig.AgentURL().String() - c.transport = newHTTPTransport(c.internalConfig.TraceURL(), agentURL, c.httpClient) + headers := datadogHeaders() + if h := c.internalConfig.OTLPHeaders(); h != nil { + headers = h + } + c.transport = newHTTPTransport(c.internalConfig.TraceURL(), agentURL, c.httpClient, headers) } if c.propagator == nil { envKey := "DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH" diff --git a/ddtrace/tracer/tracer_test.go b/ddtrace/tracer/tracer_test.go index 6b3936f0e0f..5b019f86591 100644 --- a/ddtrace/tracer/tracer_test.go +++ b/ddtrace/tracer/tracer_test.go @@ -1239,7 +1239,7 @@ func TestTracerNoDebugStack(t *testing.T) { // newDefaultTransport return a default transport for this tracing client func newDefaultTransport() transport { - return newHTTPTransport(defaultURL+tracesAPIPath, defaultURL, internal.DefaultHTTPClient(defaultHTTPTimeout, true)) + return newHTTPTransport(defaultURL+tracesAPIPath, defaultURL, internal.DefaultHTTPClient(defaultHTTPTimeout, true), datadogHeaders()) } func TestNewSpan(t *testing.T) { @@ -1358,7 +1358,7 @@ func TestTracerPrioritySampler(t *testing.T) { url := "http://" + srv.Listener.Addr().String() tr, _, flush, stop, err := startTestTracer(t, - withTransport(newHTTPTransport(url+tracesAPIPath, url, internal.DefaultHTTPClient(defaultHTTPTimeout, false))), + withTransport(newHTTPTransport(url+tracesAPIPath, url, internal.DefaultHTTPClient(defaultHTTPTimeout, false), datadogHeaders())), ) assert.Nil(err) defer stop() diff --git a/ddtrace/tracer/transport.go b/ddtrace/tracer/transport.go index 4f3e583942f..80e897251c0 100644 --- a/ddtrace/tracer/transport.go +++ b/ddtrace/tracer/transport.go @@ -65,11 +65,20 @@ type httpTransport struct { // newHTTPTransport returns a new Transport implementation that sends traces // to the given traceURL and stats to the given agentURL, using the provided -// *http.Client. The traceURL should be the fully resolved endpoint (including -// path); the agentURL is the base agent address used to derive the stats -// endpoint. -func newHTTPTransport(traceURL string, agentURL string, client *http.Client) *httpTransport { - defaultHeaders := map[string]string{ +// *http.Client and headers. The caller is responsible for providing the +// appropriate headers (e.g. datadogHeaders() for Datadog mode, or OTLP +// headers resolved from config). +func newHTTPTransport(traceURL string, agentURL string, client *http.Client, headers map[string]string) *httpTransport { + return &httpTransport{ + traceURL: traceURL, + statsURL: fmt.Sprintf("%s%s", agentURL, statsAPIPath), + client: client, + headers: headers, + } +} + +func datadogHeaders() map[string]string { + h := map[string]string{ "Datadog-Meta-Lang": "go", "Datadog-Meta-Lang-Version": strings.TrimPrefix(runtime.Version(), "go"), "Datadog-Meta-Lang-Interpreter": runtime.Compiler + "-" + runtime.GOARCH + "-" + runtime.GOOS, @@ -77,20 +86,15 @@ func newHTTPTransport(traceURL string, agentURL string, client *http.Client) *ht "Content-Type": "application/msgpack", } if cid := internal.ContainerID(); cid != "" { - defaultHeaders["Datadog-Container-ID"] = cid + h["Datadog-Container-ID"] = cid } if eid := internal.EntityID(); eid != "" { - defaultHeaders["Datadog-Entity-ID"] = eid + h["Datadog-Entity-ID"] = eid } if extEnv := internal.ExternalEnvironment(); extEnv != "" { - defaultHeaders["Datadog-External-Env"] = extEnv - } - return &httpTransport{ - traceURL: traceURL, - statsURL: fmt.Sprintf("%s%s", agentURL, statsAPIPath), - client: client, - headers: defaultHeaders, + h["Datadog-External-Env"] = extEnv } + return h } func (t *httpTransport) sendStats(p *pb.ClientStatsPayload, tracerObfuscationVersion int) error { diff --git a/ddtrace/tracer/transport_bench_test.go b/ddtrace/tracer/transport_bench_test.go index f3f3306bb1e..e747ab325e0 100644 --- a/ddtrace/tracer/transport_bench_test.go +++ b/ddtrace/tracer/transport_bench_test.go @@ -24,7 +24,7 @@ func BenchmarkHTTPTransportSend(b *testing.B) { })) defer server.Close() - transport := newHTTPTransport(server.URL+tracesAPIPath, server.URL, internal.DefaultHTTPClient(5*time.Second, false)) + transport := newHTTPTransport(server.URL+tracesAPIPath, server.URL, internal.DefaultHTTPClient(5*time.Second, false), datadogHeaders()) payloadSizes := []struct { name string @@ -69,7 +69,7 @@ func BenchmarkTransportSendConcurrent(b *testing.B) { })) defer server.Close() - transport := newHTTPTransport(server.URL+tracesAPIPath, server.URL, internal.DefaultHTTPClient(5*time.Second, false)) + transport := newHTTPTransport(server.URL+tracesAPIPath, server.URL, internal.DefaultHTTPClient(5*time.Second, false), datadogHeaders()) concurrencyLevels := []int{1, 2, 4, 8} for _, concurrency := range concurrencyLevels { diff --git a/ddtrace/tracer/transport_test.go b/ddtrace/tracer/transport_test.go index 7c5548ab222..4b46157a80a 100644 --- a/ddtrace/tracer/transport_test.go +++ b/ddtrace/tracer/transport_test.go @@ -90,7 +90,7 @@ func TestTracesAgentIntegration(t *testing.T) { } for _, tc := range testCases { - transport := newHTTPTransport(defaultURL+tracesAPIPath, defaultURL, internal.DefaultHTTPClient(defaultHTTPTimeout, false)) + transport := newHTTPTransport(defaultURL+tracesAPIPath, defaultURL, internal.DefaultHTTPClient(defaultHTTPTimeout, false), datadogHeaders()) p, err := encode(tc.payload) assert.NoError(err) body, err := transport.send(p) @@ -167,7 +167,7 @@ func TestTransportResponse(t *testing.T) { w.Write([]byte(tt.body)) })) defer srv.Close() - transport := newHTTPTransport(srv.URL+tracesAPIPath, srv.URL, internal.DefaultHTTPClient(defaultHTTPTimeout, false)) + transport := newHTTPTransport(srv.URL+tracesAPIPath, srv.URL, internal.DefaultHTTPClient(defaultHTTPTimeout, false), datadogHeaders()) rc, err := transport.send(newPayload(traceProtocolV04)) if tt.err != "" { assert.Equal(tt.err, err.Error()) @@ -207,7 +207,7 @@ func TestTraceCountHeader(t *testing.T) { })) defer srv.Close() for _, tc := range testCases { - transport := newHTTPTransport(srv.URL+tracesAPIPath, srv.URL, internal.DefaultHTTPClient(defaultHTTPTimeout, false)) + transport := newHTTPTransport(srv.URL+tracesAPIPath, srv.URL, internal.DefaultHTTPClient(defaultHTTPTimeout, false), datadogHeaders()) p, err := encode(tc.payload) assert.NoError(err) _, err = transport.send(p) @@ -249,7 +249,7 @@ func TestCustomTransport(t *testing.T) { c := &http.Client{} crt := wrapRecordingRoundTripper(c) - transport := newHTTPTransport(srv.URL+tracesAPIPath, srv.URL, c) + transport := newHTTPTransport(srv.URL+tracesAPIPath, srv.URL, c, datadogHeaders()) p, err := encode(getTestTrace(1, 1)) assert.NoError(err) _, err = transport.send(p) @@ -634,7 +634,7 @@ func TestConcurrentTraceFlushOverUDS(t *testing.T) { udsURL := internal.UnixDataSocketURL(socketPath).String() client := internal.UDSClient(socketPath, 5*time.Second) - transport := newHTTPTransport(udsURL+tracesAPIPath, udsURL, client) + transport := newHTTPTransport(udsURL+tracesAPIPath, udsURL, client, datadogHeaders()) const numGoroutines = 20 diff --git a/internal/config/config.go b/internal/config/config.go index 0ca2b49e87b..5ccafe70f89 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -114,6 +114,9 @@ type Config struct { // traceURL is the endpoint URL for sending traces, resolved from // the active traceProtocol, agent URL, and OTLP env vars. traceURL string + // otlpHeaders holds the resolved OTLP trace headers when using traceProtocolOTLP. + // nil when not in OTLP mode. Always includes Content-Type: application/x-protobuf. + otlpHeaders map[string]string } // loadConfig initializes and returns a new config by reading from all configured sources. @@ -161,6 +164,11 @@ func loadConfig() *Config { otlpTracesEndpoint := p.GetString("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") otlpEndpoint := p.GetString("OTEL_EXPORTER_OTLP_ENDPOINT", "") cfg.traceURL = resolveTraceURL(cfg.traceProtocol, cfg.agentURL, otlpTracesEndpoint, otlpEndpoint) + if cfg.traceProtocol == TraceProtocolOTLP { + otlpTracesHeaders := p.GetString("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "") + otlpFallbackHeaders := p.GetString("OTEL_EXPORTER_OTLP_HEADERS", "") + cfg.otlpHeaders = resolveOTLPHeaders(otlpTracesHeaders, otlpFallbackHeaders) + } // Parse feature flags from DD_TRACE_FEATURES as a set cfg.featureFlags = make(map[string]struct{}) @@ -714,3 +722,10 @@ func (c *Config) TraceURL() string { defer c.mu.RUnlock() return c.traceURL } + +// OTLPHeaders returns the resolved OTLP trace headers, or nil when not in OTLP mode. +func (c *Config) OTLPHeaders() map[string]string { + c.mu.RLock() + defer c.mu.RUnlock() + return maps.Clone(c.otlpHeaders) +} diff --git a/internal/config/config_helpers.go b/internal/config/config_helpers.go index 6e1345bf74d..b3c35b5ab2b 100644 --- a/internal/config/config_helpers.go +++ b/internal/config/config_helpers.go @@ -47,6 +47,9 @@ const ( // OTLP standard traces path and default collector port. otlpTracesPath = "/v1/traces" otlpDefaultPort = "4318" + + // OTLPContentTypeHeader is the Content-Type header value required for HTTP protobuf payloads. + OTLPContentTypeHeader = "application/x-protobuf" ) func validateSampleRate(rate float64) bool { @@ -188,3 +191,38 @@ func resolveOTLPTraceURL(rawAgentURL *url.URL, otlpTracesEndpoint, otlpEndpoint } return fmt.Sprintf("http://%s:%s%s", host, otlpDefaultPort, otlpTracesPath) } + +// resolveOTLPHeaders builds the OTLP trace headers map from the signal-specific +// and fallback header strings (OTEL_EXPORTER_OTLP_TRACES_HEADERS / +// OTEL_EXPORTER_OTLP_HEADERS). The signal-specific value takes full precedence. +// The required Content-Type header for HTTP protobuf is always added. +func resolveOTLPHeaders(otlpTracesHeaders, otlpHeaders string) map[string]string { + raw := otlpTracesHeaders + if raw == "" { + raw = otlpHeaders + } + headers := parseOTLPHeaders(raw) + headers["Content-Type"] = OTLPContentTypeHeader + return headers +} + +// parseOTLPHeaders parses a comma-separated list of key=value pairs as +// defined by the OTEL_EXPORTER_OTLP_*_HEADERS specification. +func parseOTLPHeaders(s string) map[string]string { + headers := make(map[string]string) + if s == "" { + return headers + } + for _, pair := range strings.Split(s, ",") { + pair = strings.TrimSpace(pair) + if pair == "" { + continue + } + k, v, ok := strings.Cut(pair, "=") + if !ok || strings.TrimSpace(k) == "" { + continue + } + headers[strings.TrimSpace(k)] = strings.TrimSpace(v) + } + return headers +} diff --git a/internal/config/config_helpers_test.go b/internal/config/config_helpers_test.go index 4b706433e5b..e460cb4ac83 100644 --- a/internal/config/config_helpers_test.go +++ b/internal/config/config_helpers_test.go @@ -82,3 +82,60 @@ func TestResolveOTLPTraceURL(t *testing.T) { assert.Equal(t, "http://localhost:4318/v1/traces", got) }) } + +func TestResolveOTLPHeaders(t *testing.T) { + t.Run("TRACES_HEADERS takes priority over HEADERS", func(t *testing.T) { + got := resolveOTLPHeaders("api-key=traces-key", "api-key=generic-key") + assert.Equal(t, "traces-key", got["api-key"]) + }) + + t.Run("falls back to HEADERS when TRACES_HEADERS is empty", func(t *testing.T) { + got := resolveOTLPHeaders("", "api-key=fallback") + assert.Equal(t, "fallback", got["api-key"]) + }) + + t.Run("always includes Content-Type for protobuf", func(t *testing.T) { + got := resolveOTLPHeaders("", "") + assert.Equal(t, OTLPContentTypeHeader, got["Content-Type"]) + }) + + t.Run("user headers plus Content-Type", func(t *testing.T) { + got := resolveOTLPHeaders("api-key=key,other=value", "") + assert.Equal(t, "key", got["api-key"]) + assert.Equal(t, "value", got["other"]) + assert.Equal(t, OTLPContentTypeHeader, got["Content-Type"]) + }) + +} + +func TestParseOTLPHeaders(t *testing.T) { + t.Run("parses comma-separated key=value pairs", func(t *testing.T) { + got := parseOTLPHeaders("api-key=key,other-config=value") + assert.Equal(t, map[string]string{"api-key": "key", "other-config": "value"}, got) + }) + + t.Run("trims whitespace around keys and values", func(t *testing.T) { + got := parseOTLPHeaders(" api-key = key , other = value ") + assert.Equal(t, map[string]string{"api-key": "key", "other": "value"}, got) + }) + + t.Run("skips empty pairs", func(t *testing.T) { + got := parseOTLPHeaders("api-key=key,,other=value,") + assert.Equal(t, map[string]string{"api-key": "key", "other": "value"}, got) + }) + + t.Run("skips pairs without =", func(t *testing.T) { + got := parseOTLPHeaders("valid=yes,invalid,also-valid=ok") + assert.Equal(t, map[string]string{"valid": "yes", "also-valid": "ok"}, got) + }) + + t.Run("empty string returns empty map", func(t *testing.T) { + got := parseOTLPHeaders("") + assert.Empty(t, got) + }) + + t.Run("value can contain equals sign", func(t *testing.T) { + got := parseOTLPHeaders("auth=base64encoded==") + assert.Equal(t, "base64encoded==", got["auth"]) + }) +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index cca9be8ca43..57e3df6d355 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -570,6 +570,77 @@ func TestDDTraceProtocolTakesPrecedenceOverOTEL(t *testing.T) { }) } +func TestOTLPHeaders(t *testing.T) { + t.Run("nil when not in OTLP mode", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + cfg := Get() + require.NotNil(t, cfg) + + assert.Nil(t, cfg.OTLPHeaders()) + }) + + t.Run("includes Content-Type in OTLP mode with no custom headers", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + + cfg := Get() + require.NotNil(t, cfg) + + headers := cfg.OTLPHeaders() + require.NotNil(t, headers) + assert.Equal(t, OTLPContentTypeHeader, headers["Content-Type"]) + }) + + t.Run("OTEL_EXPORTER_OTLP_TRACES_HEADERS parsed into map", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "api-key=secret,x-custom=value") + + cfg := Get() + require.NotNil(t, cfg) + + headers := cfg.OTLPHeaders() + assert.Equal(t, "secret", headers["api-key"]) + assert.Equal(t, "value", headers["x-custom"]) + assert.Equal(t, OTLPContentTypeHeader, headers["Content-Type"]) + }) + + t.Run("OTEL_EXPORTER_OTLP_HEADERS used as fallback", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + t.Setenv("OTEL_EXPORTER_OTLP_HEADERS", "api-key=fallback") + + cfg := Get() + require.NotNil(t, cfg) + + headers := cfg.OTLPHeaders() + assert.Equal(t, "fallback", headers["api-key"]) + }) + + t.Run("TRACES_HEADERS takes priority over HEADERS", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "api-key=specific") + t.Setenv("OTEL_EXPORTER_OTLP_HEADERS", "api-key=generic") + + cfg := Get() + require.NotNil(t, cfg) + + headers := cfg.OTLPHeaders() + assert.Equal(t, "specific", headers["api-key"]) + }) +} + func TestSetTraceProtocolRecomputesTraceURL(t *testing.T) { resetGlobalState() defer resetGlobalState() diff --git a/internal/env/supported_configurations.gen.go b/internal/env/supported_configurations.gen.go index 9bc8ba83d7a..ac670b0ff10 100644 --- a/internal/env/supported_configurations.gen.go +++ b/internal/env/supported_configurations.gen.go @@ -247,6 +247,7 @@ var SupportedConfigurations = map[string]struct{}{ "OTEL_EXPORTER_OTLP_PROTOCOL": {}, "OTEL_EXPORTER_OTLP_TIMEOUT": {}, "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": {}, + "OTEL_EXPORTER_OTLP_TRACES_HEADERS": {}, "OTEL_LOGS_EXPORTER": {}, "OTEL_LOG_LEVEL": {}, "OTEL_METRICS_EXPORTER": {}, diff --git a/internal/env/supported_configurations.json b/internal/env/supported_configurations.json index c6e2d5bc737..b9170ffded1 100644 --- a/internal/env/supported_configurations.json +++ b/internal/env/supported_configurations.json @@ -1673,6 +1673,13 @@ "default": "" } ], + "OTEL_EXPORTER_OTLP_TRACES_HEADERS": [ + { + "implementation": "B", + "type": "string", + "default": "" + } + ], "OTEL_LOGS_EXPORTER": [ { "implementation": "A", From cb6908e96f4ee010905a5977259fb9bef9aa3a53 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Mon, 23 Mar 2026 11:02:38 -0400 Subject: [PATCH 04/58] Fix superfluous referecne to unsupport OTLP header config --- ddtrace/tracer/option.go | 4 +-- internal/config/config.go | 11 +++---- internal/config/config_helpers.go | 15 +++------ internal/config/config_helpers_test.go | 19 ++++------- internal/config/config_test.go | 44 ++------------------------ 5 files changed, 20 insertions(+), 73 deletions(-) diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 3fa34b1876b..33ba4b2712c 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -429,8 +429,8 @@ func newConfig(opts ...StartOption) (*config, error) { if c.transport == nil { agentURL := c.internalConfig.AgentURL().String() headers := datadogHeaders() - if h := c.internalConfig.OTLPHeaders(); h != nil { - headers = h + if c.internalConfig.TraceProtocol() == traceProtocolOTLP { + headers = c.internalConfig.OTLPHeaders() } c.transport = newHTTPTransport(c.internalConfig.TraceURL(), agentURL, c.httpClient, headers) } diff --git a/internal/config/config.go b/internal/config/config.go index 5ccafe70f89..adb3558913d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -114,8 +114,9 @@ type Config struct { // traceURL is the endpoint URL for sending traces, resolved from // the active traceProtocol, agent URL, and OTLP env vars. traceURL string - // otlpHeaders holds the resolved OTLP trace headers when using traceProtocolOTLP. - // nil when not in OTLP mode. Always includes Content-Type: application/x-protobuf. + // otlpHeaders holds the resolved OTLP trace headers from + // OTEL_EXPORTER_OTLP_TRACES_HEADERS plus Content-Type: application/x-protobuf. + // Always populated; the tracer decides whether to use them based on protocol. otlpHeaders map[string]string } @@ -164,11 +165,7 @@ func loadConfig() *Config { otlpTracesEndpoint := p.GetString("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") otlpEndpoint := p.GetString("OTEL_EXPORTER_OTLP_ENDPOINT", "") cfg.traceURL = resolveTraceURL(cfg.traceProtocol, cfg.agentURL, otlpTracesEndpoint, otlpEndpoint) - if cfg.traceProtocol == TraceProtocolOTLP { - otlpTracesHeaders := p.GetString("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "") - otlpFallbackHeaders := p.GetString("OTEL_EXPORTER_OTLP_HEADERS", "") - cfg.otlpHeaders = resolveOTLPHeaders(otlpTracesHeaders, otlpFallbackHeaders) - } + cfg.otlpHeaders = resolveOTLPHeaders(p.GetString("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "")) // Parse feature flags from DD_TRACE_FEATURES as a set cfg.featureFlags = make(map[string]struct{}) diff --git a/internal/config/config_helpers.go b/internal/config/config_helpers.go index b3c35b5ab2b..e600d4bf20d 100644 --- a/internal/config/config_helpers.go +++ b/internal/config/config_helpers.go @@ -192,16 +192,11 @@ func resolveOTLPTraceURL(rawAgentURL *url.URL, otlpTracesEndpoint, otlpEndpoint return fmt.Sprintf("http://%s:%s%s", host, otlpDefaultPort, otlpTracesPath) } -// resolveOTLPHeaders builds the OTLP trace headers map from the signal-specific -// and fallback header strings (OTEL_EXPORTER_OTLP_TRACES_HEADERS / -// OTEL_EXPORTER_OTLP_HEADERS). The signal-specific value takes full precedence. -// The required Content-Type header for HTTP protobuf is always added. -func resolveOTLPHeaders(otlpTracesHeaders, otlpHeaders string) map[string]string { - raw := otlpTracesHeaders - if raw == "" { - raw = otlpHeaders - } - headers := parseOTLPHeaders(raw) +// resolveOTLPHeaders builds the OTLP trace headers map from +// OTEL_EXPORTER_OTLP_TRACES_HEADERS. The required Content-Type header for +// HTTP protobuf is always included. +func resolveOTLPHeaders(otlpTracesHeaders string) map[string]string { + headers := parseOTLPHeaders(otlpTracesHeaders) headers["Content-Type"] = OTLPContentTypeHeader return headers } diff --git a/internal/config/config_helpers_test.go b/internal/config/config_helpers_test.go index e460cb4ac83..1173fcf7d0d 100644 --- a/internal/config/config_helpers_test.go +++ b/internal/config/config_helpers_test.go @@ -84,28 +84,23 @@ func TestResolveOTLPTraceURL(t *testing.T) { } func TestResolveOTLPHeaders(t *testing.T) { - t.Run("TRACES_HEADERS takes priority over HEADERS", func(t *testing.T) { - got := resolveOTLPHeaders("api-key=traces-key", "api-key=generic-key") - assert.Equal(t, "traces-key", got["api-key"]) - }) - - t.Run("falls back to HEADERS when TRACES_HEADERS is empty", func(t *testing.T) { - got := resolveOTLPHeaders("", "api-key=fallback") - assert.Equal(t, "fallback", got["api-key"]) - }) - t.Run("always includes Content-Type for protobuf", func(t *testing.T) { - got := resolveOTLPHeaders("", "") + got := resolveOTLPHeaders("") assert.Equal(t, OTLPContentTypeHeader, got["Content-Type"]) }) t.Run("user headers plus Content-Type", func(t *testing.T) { - got := resolveOTLPHeaders("api-key=key,other=value", "") + got := resolveOTLPHeaders("api-key=key,other=value") assert.Equal(t, "key", got["api-key"]) assert.Equal(t, "value", got["other"]) assert.Equal(t, OTLPContentTypeHeader, got["Content-Type"]) }) + t.Run("empty TRACES_HEADERS still gets Content-Type", func(t *testing.T) { + got := resolveOTLPHeaders("") + assert.Len(t, got, 1) + assert.Equal(t, OTLPContentTypeHeader, got["Content-Type"]) + }) } func TestParseOTLPHeaders(t *testing.T) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 57e3df6d355..3e619eec5e7 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -571,35 +571,23 @@ func TestDDTraceProtocolTakesPrecedenceOverOTEL(t *testing.T) { } func TestOTLPHeaders(t *testing.T) { - t.Run("nil when not in OTLP mode", func(t *testing.T) { + t.Run("always populated with at least Content-Type", func(t *testing.T) { resetGlobalState() defer resetGlobalState() cfg := Get() require.NotNil(t, cfg) - assert.Nil(t, cfg.OTLPHeaders()) - }) - - t.Run("includes Content-Type in OTLP mode with no custom headers", func(t *testing.T) { - resetGlobalState() - defer resetGlobalState() - - t.Setenv("OTEL_TRACES_EXPORTER", "otlp") - - cfg := Get() - require.NotNil(t, cfg) - headers := cfg.OTLPHeaders() require.NotNil(t, headers) assert.Equal(t, OTLPContentTypeHeader, headers["Content-Type"]) + assert.Len(t, headers, 1) }) t.Run("OTEL_EXPORTER_OTLP_TRACES_HEADERS parsed into map", func(t *testing.T) { resetGlobalState() defer resetGlobalState() - t.Setenv("OTEL_TRACES_EXPORTER", "otlp") t.Setenv("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "api-key=secret,x-custom=value") cfg := Get() @@ -611,34 +599,6 @@ func TestOTLPHeaders(t *testing.T) { assert.Equal(t, OTLPContentTypeHeader, headers["Content-Type"]) }) - t.Run("OTEL_EXPORTER_OTLP_HEADERS used as fallback", func(t *testing.T) { - resetGlobalState() - defer resetGlobalState() - - t.Setenv("OTEL_TRACES_EXPORTER", "otlp") - t.Setenv("OTEL_EXPORTER_OTLP_HEADERS", "api-key=fallback") - - cfg := Get() - require.NotNil(t, cfg) - - headers := cfg.OTLPHeaders() - assert.Equal(t, "fallback", headers["api-key"]) - }) - - t.Run("TRACES_HEADERS takes priority over HEADERS", func(t *testing.T) { - resetGlobalState() - defer resetGlobalState() - - t.Setenv("OTEL_TRACES_EXPORTER", "otlp") - t.Setenv("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "api-key=specific") - t.Setenv("OTEL_EXPORTER_OTLP_HEADERS", "api-key=generic") - - cfg := Get() - require.NotNil(t, cfg) - - headers := cfg.OTLPHeaders() - assert.Equal(t, "specific", headers["api-key"]) - }) } func TestSetTraceProtocolRecomputesTraceURL(t *testing.T) { From a85fedf4afcb56968208169594eadd5effd0e8e9 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Mon, 23 Mar 2026 11:17:06 -0400 Subject: [PATCH 05/58] Introduce otlpExportMode on Config; use this instead of traceProtocol to determine otlp mode --- ddtrace/tracer/option.go | 7 ++-- internal/config/config.go | 29 ++++++++++----- internal/config/config_helpers.go | 36 ++++++++----------- internal/config/config_helpers_test.go | 18 +++++----- internal/config/config_test.go | 10 +++--- .../config/provider/otelenvconfigsource.go | 16 +-------- .../provider/otelenvconfigsource_test.go | 14 -------- 7 files changed, 54 insertions(+), 76 deletions(-) diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 33ba4b2712c..8c03dabeeb9 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -134,9 +134,8 @@ var ( // Supported trace protocols. const ( - traceProtocolV04 = internalconfig.TraceProtocolV04 - traceProtocolV1 = internalconfig.TraceProtocolV1 - traceProtocolOTLP = internalconfig.TraceProtocolOTLP + traceProtocolV04 = internalconfig.TraceProtocolV04 + traceProtocolV1 = internalconfig.TraceProtocolV1 ) // config holds the tracer configuration. @@ -429,7 +428,7 @@ func newConfig(opts ...StartOption) (*config, error) { if c.transport == nil { agentURL := c.internalConfig.AgentURL().String() headers := datadogHeaders() - if c.internalConfig.TraceProtocol() == traceProtocolOTLP { + if c.internalConfig.OTLPExportMode() { headers = c.internalConfig.OTLPHeaders() } c.transport = newHTTPTransport(c.internalConfig.TraceURL(), agentURL, c.httpClient, headers) diff --git a/internal/config/config.go b/internal/config/config.go index adb3558913d..7a09cc5fde5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -107,16 +107,17 @@ type Config struct { retryInterval time.Duration // logsOTelEnabled controls if the OpenTelemetry Logs SDK pipeline should be enabled logsOTelEnabled bool - // traceProtocol is the trace protocol version to use (e.g. TraceProtocolV04 or TraceProtocolV1). - // Initialized from DD_TRACE_AGENT_PROTOCOL_VERSION, may be downgraded by the tracer - // if the agent doesn't support the requested version. + // traceProtocol is the Datadog trace protocol version (TraceProtocolV04 or TraceProtocolV1). + // Only meaningful when otlpExportMode is false. traceProtocol float64 - // traceURL is the endpoint URL for sending traces, resolved from - // the active traceProtocol, agent URL, and OTLP env vars. + // otlpExportMode indicates traces should be exported via OTLP rather than + // a Datadog protocol. Set when OTEL_TRACES_EXPORTER=otlp. + otlpExportMode bool + // traceURL is the endpoint URL for sending traces. traceURL string // otlpHeaders holds the resolved OTLP trace headers from // OTEL_EXPORTER_OTLP_TRACES_HEADERS plus Content-Type: application/x-protobuf. - // Always populated; the tracer decides whether to use them based on protocol. + // Always populated; the tracer decides whether to use them based on otlpExportMode. otlpHeaders map[string]string } @@ -162,9 +163,14 @@ func loadConfig() *Config { cfg.retryInterval = p.GetDuration("DD_TRACE_RETRY_INTERVAL", time.Millisecond) cfg.logsOTelEnabled = p.GetBool("DD_LOGS_OTEL_ENABLED", false) cfg.traceProtocol = resolveTraceProtocol(p.GetStringWithValidator("DD_TRACE_AGENT_PROTOCOL_VERSION", "0.4", validateTraceProtocolVersion)) + cfg.otlpExportMode = p.GetString("OTEL_TRACES_EXPORTER", "") == "otlp" + // DD_TRACE_AGENT_PROTOCOL_VERSION takes precedence and disables OTLP export. + if cfg.otlpExportMode && env.Get("DD_TRACE_AGENT_PROTOCOL_VERSION") != "" { + cfg.otlpExportMode = false + } otlpTracesEndpoint := p.GetString("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") otlpEndpoint := p.GetString("OTEL_EXPORTER_OTLP_ENDPOINT", "") - cfg.traceURL = resolveTraceURL(cfg.traceProtocol, cfg.agentURL, otlpTracesEndpoint, otlpEndpoint) + cfg.traceURL = resolveTraceURL(cfg.otlpExportMode, cfg.traceProtocol, cfg.agentURL, otlpTracesEndpoint, otlpEndpoint) cfg.otlpHeaders = resolveOTLPHeaders(p.GetString("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "")) // Parse feature flags from DD_TRACE_FEATURES as a set @@ -710,7 +716,7 @@ func (c *Config) SetTraceProtocol(v float64, origin telemetry.Origin) { c.mu.Lock() defer c.mu.Unlock() c.traceProtocol = v - c.traceURL = resolveTraceURL(v, c.agentURL, "", "") + c.traceURL = resolveTraceURL(c.otlpExportMode, v, c.agentURL, "", "") configtelemetry.Report("DD_TRACE_AGENT_PROTOCOL_VERSION", v, origin) } @@ -720,7 +726,12 @@ func (c *Config) TraceURL() string { return c.traceURL } -// OTLPHeaders returns the resolved OTLP trace headers, or nil when not in OTLP mode. +func (c *Config) OTLPExportMode() bool { + c.mu.RLock() + defer c.mu.RUnlock() + return c.otlpExportMode +} + func (c *Config) OTLPHeaders() map[string]string { c.mu.RLock() defer c.mu.RUnlock() diff --git a/internal/config/config_helpers.go b/internal/config/config_helpers.go index e600d4bf20d..42f5ca62386 100644 --- a/internal/config/config_helpers.go +++ b/internal/config/config_helpers.go @@ -27,13 +27,11 @@ const ( // nothing. TraceMaxSize = int(1e5) - // Trace protocol versions (agent wire format). - TraceProtocolV04 = 0.4 // default - TraceProtocolV1 = 1.0 - TraceProtocolOTLP = 2.0 - TraceProtocolVersionStringV04 = "0.4" - TraceProtocolVersionStringV1 = "1.0" - TraceProtocolVersionStringOTLP = "otlp" + // Datadog trace protocol versions (agent wire format). + TraceProtocolV04 = 0.4 // default + TraceProtocolV1 = 1.0 + TraceProtocolVersionStringV04 = "0.4" + TraceProtocolVersionStringV1 = "1.0" // Agent URL schemes supported by DD_TRACE_AGENT_URL. URLSchemeUnix = "unix" @@ -81,7 +79,7 @@ func validatePartialFlushMinSpans(minSpans int) bool { } func validateTraceProtocolVersion(v string) bool { - return v == TraceProtocolVersionStringV04 || v == TraceProtocolVersionStringV1 || v == TraceProtocolVersionStringOTLP + return v == TraceProtocolVersionStringV04 || v == TraceProtocolVersionStringV1 } func resolveTraceProtocol(v string) float64 { @@ -90,8 +88,6 @@ func resolveTraceProtocol(v string) float64 { return TraceProtocolV04 case TraceProtocolVersionStringV1: return TraceProtocolV1 - case TraceProtocolVersionStringOTLP: - return TraceProtocolOTLP default: return TraceProtocolV04 } @@ -152,23 +148,21 @@ func detectUDSURL() *url.URL { } } -// resolveTraceURL computes the full trace endpoint URL based on the protocol -// and agent URL. For Datadog protocols the URL is derived from agentURL; for -// OTLP it uses the provided endpoint values (sourced from the config provider). -func resolveTraceURL(protocol float64, rawAgentURL *url.URL, otlpTracesEndpoint, otlpEndpoint string) string { +// resolveTraceURL computes the full trace endpoint URL. In OTLP export mode +// it uses the OTLP endpoint values; otherwise it derives the URL from the +// agent URL and Datadog protocol version. +func resolveTraceURL(otlpMode bool, protocol float64, rawAgentURL *url.URL, otlpTracesEndpoint, otlpEndpoint string) string { + if otlpMode { + return resolveOTLPTraceURL(rawAgentURL, otlpTracesEndpoint, otlpEndpoint) + } agentHTTPURL := rawAgentURL if rawAgentURL != nil && rawAgentURL.Scheme == URLSchemeUnix { agentHTTPURL = internal.UnixDataSocketURL(rawAgentURL.Path) } - - switch protocol { - case TraceProtocolOTLP: - return resolveOTLPTraceURL(rawAgentURL, otlpTracesEndpoint, otlpEndpoint) - case TraceProtocolV1: + if protocol == TraceProtocolV1 { return agentHTTPURL.String() + TracesPathV1 - default: - return agentHTTPURL.String() + TracesPathV04 } + return agentHTTPURL.String() + TracesPathV04 } // resolveOTLPTraceURL resolves the OTLP trace endpoint using the standard diff --git a/internal/config/config_helpers_test.go b/internal/config/config_helpers_test.go index 1173fcf7d0d..fad1b9feff4 100644 --- a/internal/config/config_helpers_test.go +++ b/internal/config/config_helpers_test.go @@ -17,33 +17,33 @@ func TestResolveTraceURL(t *testing.T) { unixAgent := &url.URL{Scheme: "unix", Path: "/var/run/datadog/apm.socket"} t.Run("v0.4 protocol uses agent URL", func(t *testing.T) { - got := resolveTraceURL(TraceProtocolV04, httpAgent, "", "") + got := resolveTraceURL(false, TraceProtocolV04, httpAgent, "", "") assert.Equal(t, "http://myhost:8126/v0.4/traces", got) }) t.Run("v1 protocol uses agent URL", func(t *testing.T) { - got := resolveTraceURL(TraceProtocolV1, httpAgent, "", "") + got := resolveTraceURL(false, TraceProtocolV1, httpAgent, "", "") assert.Equal(t, "http://myhost:8126/v1.0/traces", got) }) t.Run("v0.4 with unix socket rewrites to HTTP", func(t *testing.T) { - got := resolveTraceURL(TraceProtocolV04, unixAgent, "", "") + got := resolveTraceURL(false, TraceProtocolV04, unixAgent, "", "") assert.Contains(t, got, "/v0.4/traces") assert.Contains(t, got, "http://") }) - t.Run("OTLP delegates to resolveOTLPTraceURL", func(t *testing.T) { - got := resolveTraceURL(TraceProtocolOTLP, httpAgent, "", "") + t.Run("OTLP mode delegates to resolveOTLPTraceURL", func(t *testing.T) { + got := resolveTraceURL(true, TraceProtocolV04, httpAgent, "", "") assert.Equal(t, "http://myhost:4318/v1/traces", got) }) - t.Run("OTLP with traces endpoint set", func(t *testing.T) { - got := resolveTraceURL(TraceProtocolOTLP, httpAgent, "http://collector:4318/v1/traces", "") + t.Run("OTLP mode with traces endpoint set", func(t *testing.T) { + got := resolveTraceURL(true, TraceProtocolV04, httpAgent, "http://collector:4318/v1/traces", "") assert.Equal(t, "http://collector:4318/v1/traces", got) }) - t.Run("OTLP with base endpoint set", func(t *testing.T) { - got := resolveTraceURL(TraceProtocolOTLP, httpAgent, "", "http://collector:4318") + t.Run("OTLP mode with base endpoint set", func(t *testing.T) { + got := resolveTraceURL(true, TraceProtocolV04, httpAgent, "", "http://collector:4318") assert.Equal(t, "http://collector:4318/v1/traces", got) }) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 3e619eec5e7..076c23321d4 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -473,7 +473,7 @@ func TestTraceURLResolution(t *testing.T) { assert.Contains(t, traceURL, "/v0.4/traces") }) - t.Run("OTLP protocol resolves traceURL with default OTLP port", func(t *testing.T) { + t.Run("OTLP export mode resolves traceURL with default OTLP port", func(t *testing.T) { resetGlobalState() defer resetGlobalState() @@ -482,7 +482,7 @@ func TestTraceURLResolution(t *testing.T) { cfg := Get() require.NotNil(t, cfg) - assert.Equal(t, TraceProtocolOTLP, cfg.TraceProtocol()) + assert.True(t, cfg.OTLPExportMode()) assert.Contains(t, cfg.TraceURL(), ":4318/v1/traces") }) @@ -541,7 +541,7 @@ func TestTraceURLResolution(t *testing.T) { } func TestDDTraceProtocolTakesPrecedenceOverOTEL(t *testing.T) { - t.Run("DD_TRACE_AGENT_PROTOCOL_VERSION overrides OTEL_TRACES_EXPORTER=otlp", func(t *testing.T) { + t.Run("DD_TRACE_AGENT_PROTOCOL_VERSION disables OTLP export mode", func(t *testing.T) { resetGlobalState() defer resetGlobalState() @@ -551,11 +551,12 @@ func TestDDTraceProtocolTakesPrecedenceOverOTEL(t *testing.T) { cfg := Get() require.NotNil(t, cfg) + assert.False(t, cfg.OTLPExportMode()) assert.Equal(t, TraceProtocolV04, cfg.TraceProtocol()) assert.Contains(t, cfg.TraceURL(), "/v0.4/traces") }) - t.Run("DD_TRACE_AGENT_PROTOCOL_VERSION=1.0 overrides OTEL_TRACES_EXPORTER=otlp", func(t *testing.T) { + t.Run("DD_TRACE_AGENT_PROTOCOL_VERSION=1.0 disables OTLP export mode", func(t *testing.T) { resetGlobalState() defer resetGlobalState() @@ -565,6 +566,7 @@ func TestDDTraceProtocolTakesPrecedenceOverOTEL(t *testing.T) { cfg := Get() require.NotNil(t, cfg) + assert.False(t, cfg.OTLPExportMode()) assert.Equal(t, TraceProtocolV1, cfg.TraceProtocol()) assert.Contains(t, cfg.TraceURL(), "/v1.0/traces") }) diff --git a/internal/config/provider/otelenvconfigsource.go b/internal/config/provider/otelenvconfigsource.go index d95c7565441..c1b7700804b 100644 --- a/internal/config/provider/otelenvconfigsource.go +++ b/internal/config/provider/otelenvconfigsource.go @@ -85,10 +85,6 @@ var otelConfigs = map[string]*otelDDEnv{ ot: "OTEL_RESOURCE_ATTRIBUTES", remapper: mapDDTags, }, - "DD_TRACE_AGENT_PROTOCOL_VERSION": { - ot: "OTEL_TRACES_EXPORTER", - remapper: mapTraceProtocol, - }, } var ddTagsMapping = map[string]string{ @@ -139,7 +135,7 @@ func mapEnabled(ot string) (string, error) { case "none": return "false", nil case "otlp": - return "true", nil // Handled separately by mapTraceProtocol + return "true", nil default: return "", fmt.Errorf("the following configuration is not supported: OTEL_TRACES_EXPORTER=%v", ot) } @@ -207,13 +203,3 @@ func mapDDTags(ot string) (string, error) { return strings.Join(ddTags, ","), nil } -func mapTraceProtocol(ot string) (string, error) { - switch strings.TrimSpace(strings.ToLower(ot)) { - case "none": - return "", nil // not relevant to protocol - case "otlp": - return "otlp", nil - default: - return "", fmt.Errorf("the following configuration is not supported: OTEL_TRACES_EXPORTER=%v", ot) - } -} diff --git a/internal/config/provider/otelenvconfigsource_test.go b/internal/config/provider/otelenvconfigsource_test.go index 2931b6fe98a..dd070097da8 100644 --- a/internal/config/provider/otelenvconfigsource_test.go +++ b/internal/config/provider/otelenvconfigsource_test.go @@ -84,20 +84,6 @@ func TestOtelEnvConfigSource(t *testing.T) { assert.Equal(t, "true", v) }) - t.Run("maps OTEL_TRACES_EXPORTER=otlp to DD_TRACE_AGENT_PROTOCOL_VERSION=otlp", func(t *testing.T) { - t.Setenv("OTEL_TRACES_EXPORTER", "otlp") - source := &otelEnvConfigSource{} - v := source.get("DD_TRACE_AGENT_PROTOCOL_VERSION") - assert.Equal(t, "otlp", v) - }) - - t.Run("OTEL_TRACES_EXPORTER=none does not set protocol", func(t *testing.T) { - t.Setenv("OTEL_TRACES_EXPORTER", "none") - source := &otelEnvConfigSource{} - v := source.get("DD_TRACE_AGENT_PROTOCOL_VERSION") - assert.Equal(t, "", v) - }) - t.Run("returns empty for invalid OTEL_TRACES_EXPORTER", func(t *testing.T) { telemetryClient := new(telemetrytest.RecordClient) defer telemetry.MockClient(telemetryClient)() From 33a9a3796ed1dfa71eacf00500916051b0dfb8ec Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Mon, 23 Mar 2026 11:21:55 -0400 Subject: [PATCH 06/58] fix relationship between otlpexportmode and protocol --- internal/config/config.go | 6 +----- internal/config/config_test.go | 32 -------------------------------- 2 files changed, 1 insertion(+), 37 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 7a09cc5fde5..17dac9f8600 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -162,12 +162,8 @@ func loadConfig() *Config { cfg.debugStack = p.GetBool("DD_TRACE_DEBUG_STACK", true) cfg.retryInterval = p.GetDuration("DD_TRACE_RETRY_INTERVAL", time.Millisecond) cfg.logsOTelEnabled = p.GetBool("DD_LOGS_OTEL_ENABLED", false) - cfg.traceProtocol = resolveTraceProtocol(p.GetStringWithValidator("DD_TRACE_AGENT_PROTOCOL_VERSION", "0.4", validateTraceProtocolVersion)) + cfg.traceProtocol = resolveTraceProtocol(p.GetStringWithValidator("DD_TRACE_AGENT_PROTOCOL_VERSION", TraceProtocolVersionStringV04, validateTraceProtocolVersion)) cfg.otlpExportMode = p.GetString("OTEL_TRACES_EXPORTER", "") == "otlp" - // DD_TRACE_AGENT_PROTOCOL_VERSION takes precedence and disables OTLP export. - if cfg.otlpExportMode && env.Get("DD_TRACE_AGENT_PROTOCOL_VERSION") != "" { - cfg.otlpExportMode = false - } otlpTracesEndpoint := p.GetString("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") otlpEndpoint := p.GetString("OTEL_EXPORTER_OTLP_ENDPOINT", "") cfg.traceURL = resolveTraceURL(cfg.otlpExportMode, cfg.traceProtocol, cfg.agentURL, otlpTracesEndpoint, otlpEndpoint) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 076c23321d4..3a9c468db4f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -540,38 +540,6 @@ func TestTraceURLResolution(t *testing.T) { }) } -func TestDDTraceProtocolTakesPrecedenceOverOTEL(t *testing.T) { - t.Run("DD_TRACE_AGENT_PROTOCOL_VERSION disables OTLP export mode", func(t *testing.T) { - resetGlobalState() - defer resetGlobalState() - - t.Setenv("OTEL_TRACES_EXPORTER", "otlp") - t.Setenv("DD_TRACE_AGENT_PROTOCOL_VERSION", "0.4") - - cfg := Get() - require.NotNil(t, cfg) - - assert.False(t, cfg.OTLPExportMode()) - assert.Equal(t, TraceProtocolV04, cfg.TraceProtocol()) - assert.Contains(t, cfg.TraceURL(), "/v0.4/traces") - }) - - t.Run("DD_TRACE_AGENT_PROTOCOL_VERSION=1.0 disables OTLP export mode", func(t *testing.T) { - resetGlobalState() - defer resetGlobalState() - - t.Setenv("OTEL_TRACES_EXPORTER", "otlp") - t.Setenv("DD_TRACE_AGENT_PROTOCOL_VERSION", "1.0") - - cfg := Get() - require.NotNil(t, cfg) - - assert.False(t, cfg.OTLPExportMode()) - assert.Equal(t, TraceProtocolV1, cfg.TraceProtocol()) - assert.Contains(t, cfg.TraceURL(), "/v1.0/traces") - }) -} - func TestOTLPHeaders(t *testing.T) { t.Run("always populated with at least Content-Type", func(t *testing.T) { resetGlobalState() From 3e51fdd0c3c93aa070ef09082073e1cce3573582 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Mon, 23 Mar 2026 11:42:53 -0400 Subject: [PATCH 07/58] replace tracer.config.otlpExportMode with internalconfig otlpExportMode Made-with: Cursor --- ddtrace/tracer/option.go | 3 -- ddtrace/tracer/sampler_test.go | 12 +++--- ddtrace/tracer/tracer.go | 2 +- ddtrace/tracer/tracer_test.go | 14 +++++- internal/config/config.go | 7 +++ internal/config/config_test.go | 78 ++++++++++++++++++++++++++++++++++ 6 files changed, 105 insertions(+), 11 deletions(-) diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 8c03dabeeb9..98dc1996ae1 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -259,9 +259,6 @@ type config struct { // llmobs contains the LLM Observability config llmobs llmobsconfig.Config - - // temporary field to represent otlp export mode; always false for now - otlpExportMode bool } // orchestrionConfig contains Orchestrion configuration. diff --git a/ddtrace/tracer/sampler_test.go b/ddtrace/tracer/sampler_test.go index 05d2b77b9b6..5e990adae9c 100644 --- a/ddtrace/tracer/sampler_test.go +++ b/ddtrace/tracer/sampler_test.go @@ -267,7 +267,7 @@ func TestPrioritySampler(t *testing.T) { func TestOtelParentBasedAlwaysOnSampler(t *testing.T) { t.Run("no parent keeps at rate 1.0", func(t *testing.T) { assert := assert.New(t) - tracer, err := newUnstartedTracer(func(c *config) { c.otlpExportMode = true }) + tracer, err := newUnstartedTracer(func(c *config) { c.internalConfig.SetOTLPExportMode(true, internalconfig.OriginCode) }) assert.NoError(err) defer tracer.Stop() for _, id := range []uint64{0, 1, math.MaxUint64 / 2, math.MaxUint64} { @@ -285,7 +285,7 @@ func TestOtelParentBasedAlwaysOnSampler(t *testing.T) { t.Run("inherits parent keep", func(t *testing.T) { assert := assert.New(t) - tracer, err := newUnstartedTracer(func(c *config) { c.otlpExportMode = true }) + tracer, err := newUnstartedTracer(func(c *config) { c.internalConfig.SetOTLPExportMode(true, internalconfig.OriginCode) }) assert.NoError(err) defer tracer.Stop() span := newBasicSpan("http.request") @@ -298,7 +298,7 @@ func TestOtelParentBasedAlwaysOnSampler(t *testing.T) { t.Run("inherits parent drop", func(t *testing.T) { assert := assert.New(t) - tracer, err := newUnstartedTracer(func(c *config) { c.otlpExportMode = true }) + tracer, err := newUnstartedTracer(func(c *config) { c.internalConfig.SetOTLPExportMode(true, internalconfig.OriginCode) }) assert.NoError(err) defer tracer.Stop() span := newBasicSpan("http.request") @@ -312,7 +312,7 @@ func TestOtelParentBasedAlwaysOnSampler(t *testing.T) { t.Run("DD_TRACE_SAMPLE_RATE takes precedence", func(t *testing.T) { assert := assert.New(t) t.Setenv("DD_TRACE_SAMPLE_RATE", "0") - tracer, err := newUnstartedTracer(func(c *config) { c.otlpExportMode = true }) + tracer, err := newUnstartedTracer(func(c *config) { c.internalConfig.SetOTLPExportMode(true, internalconfig.OriginCode) }) assert.NoError(err) defer tracer.Stop() span := newBasicSpan("http.request") @@ -328,7 +328,7 @@ func TestOtelParentBasedAlwaysOnSampler(t *testing.T) { assert := assert.New(t) t.Setenv("DD_TRACE_SAMPLING_RULES", `[{"service":"drop-me","sample_rate":0}]`) - tracer, err := newUnstartedTracer(func(c *config) { c.otlpExportMode = true }) + tracer, err := newUnstartedTracer(func(c *config) { c.internalConfig.SetOTLPExportMode(true, internalconfig.OriginCode) }) assert.NoError(err) defer tracer.Stop() span := newBasicSpan("http.request") @@ -345,7 +345,7 @@ func TestOtelParentBasedAlwaysOnSampler(t *testing.T) { assert := assert.New(t) t.Setenv("DD_TRACE_SAMPLING_RULES", `[{"service":"other-service","sample_rate":0}]`) - tracer, err := newUnstartedTracer(func(c *config) { c.otlpExportMode = true }) + tracer, err := newUnstartedTracer(func(c *config) { c.internalConfig.SetOTLPExportMode(true, internalconfig.OriginCode) }) assert.NoError(err) defer tracer.Stop() span := newBasicSpan("http.request") diff --git a/ddtrace/tracer/tracer.go b/ddtrace/tracer/tracer.go index 05604bfbb52..47b180ae846 100644 --- a/ddtrace/tracer/tracer.go +++ b/ddtrace/tracer/tracer.go @@ -425,7 +425,7 @@ func newUnstartedTracer(opts ...StartOption) (t *tracer, err error) { writer = newCiVisibilityTraceWriter(c) } else if c.internalConfig.LogToStdout() { writer = newLogTraceWriter(c, statsd) - } else if c.otlpExportMode { + } else if c.internalConfig.OTLPExportMode() { dfltSampler = newOtelParentBasedAlwaysOnSampler() writer = newOTLPTraceWriter(c) } else { diff --git a/ddtrace/tracer/tracer_test.go b/ddtrace/tracer/tracer_test.go index 5b019f86591..1af26f9197e 100644 --- a/ddtrace/tracer/tracer_test.go +++ b/ddtrace/tracer/tracer_test.go @@ -1471,7 +1471,7 @@ func TestTracerEdgeSampler(t *testing.T) { func TestOTLPExportMode(t *testing.T) { t.Run("uses otlpTraceWriter and otelParentBasedAlwaysOnSampler", func(t *testing.T) { assert := assert.New(t) - tracer, err := newUnstartedTracer(func(c *config) { c.otlpExportMode = true }) + tracer, err := newUnstartedTracer(func(c *config) { c.internalConfig.SetOTLPExportMode(true, internalconfig.OriginCode) }) assert.NoError(err) defer tracer.Stop() _, isOTLPWriter := tracer.traceWriter.(*otlpTraceWriter) @@ -1490,6 +1490,18 @@ func TestOTLPExportMode(t *testing.T) { _, isPriority := tracer.defaultSampler.(*prioritySampler) assert.True(isPriority, "expected prioritySampler in default mode") }) + + t.Run("OTEL_TRACES_EXPORTER=otlp env var enables OTLP mode", func(t *testing.T) { + assert := assert.New(t) + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + tracer, err := newUnstartedTracer() + assert.NoError(err) + defer tracer.Stop() + _, isOTLPWriter := tracer.traceWriter.(*otlpTraceWriter) + assert.True(isOTLPWriter, "expected otlpTraceWriter when OTEL_TRACES_EXPORTER=otlp") + _, isAlwaysOn := tracer.defaultSampler.(*otelParentBasedAlwaysOnSampler) + assert.True(isAlwaysOn, "expected otelParentBasedAlwaysOnSampler when OTEL_TRACES_EXPORTER=otlp") + }) } func TestTracerConcurrent(t *testing.T) { diff --git a/internal/config/config.go b/internal/config/config.go index 17dac9f8600..fb94aeb3681 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -728,6 +728,13 @@ func (c *Config) OTLPExportMode() bool { return c.otlpExportMode } +func (c *Config) SetOTLPExportMode(v bool, origin telemetry.Origin) { + c.mu.Lock() + defer c.mu.Unlock() + c.otlpExportMode = v + configtelemetry.Report("OTEL_TRACES_EXPORTER", v, origin) +} + func (c *Config) OTLPHeaders() map[string]string { c.mu.RLock() defer c.mu.RUnlock() diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 3a9c468db4f..ca102e431ff 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -590,6 +590,84 @@ func TestSetTraceProtocolRecomputesTraceURL(t *testing.T) { assert.Contains(t, cfg.TraceURL(), "/v0.4/traces") } +func TestOTLPExportMode(t *testing.T) { + t.Run("disabled by default", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + cfg := Get() + require.NotNil(t, cfg) + + assert.False(t, cfg.OTLPExportMode()) + }) + + t.Run("enabled by OTEL_TRACES_EXPORTER=otlp", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + + cfg := Get() + require.NotNil(t, cfg) + + assert.True(t, cfg.OTLPExportMode()) + }) + + t.Run("not enabled by unsupported exporter value", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + t.Setenv("OTEL_TRACES_EXPORTER", "jaeger") + + cfg := Get() + require.NotNil(t, cfg) + + assert.False(t, cfg.OTLPExportMode()) + }) + + t.Run("not enabled by OTEL_TRACES_EXPORTER=none", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + t.Setenv("OTEL_TRACES_EXPORTER", "none") + + cfg := Get() + require.NotNil(t, cfg) + + assert.False(t, cfg.OTLPExportMode()) + }) + + t.Run("independent of traceProtocol", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + t.Setenv("DD_TRACE_AGENT_PROTOCOL_VERSION", "1.0") + + cfg := Get() + require.NotNil(t, cfg) + + assert.True(t, cfg.OTLPExportMode(), "otlpExportMode should be true regardless of traceProtocol") + assert.Equal(t, TraceProtocolV1, cfg.TraceProtocol(), "traceProtocol should reflect DD_TRACE_AGENT_PROTOCOL_VERSION") + }) + + t.Run("SetOTLPExportMode toggles mode", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + cfg := Get() + require.NotNil(t, cfg) + + assert.False(t, cfg.OTLPExportMode()) + + cfg.SetOTLPExportMode(true, telemetry.OriginCode) + assert.True(t, cfg.OTLPExportMode()) + + cfg.SetOTLPExportMode(false, telemetry.OriginCode) + assert.False(t, cfg.OTLPExportMode()) + }) +} + func TestHostnameConfiguration(t *testing.T) { t.Run("default behavior - hostname empty when not configured", func(t *testing.T) { resetGlobalState() From 5e252befd677202359623c0399845d06565c5e7a Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Mon, 23 Mar 2026 11:51:23 -0400 Subject: [PATCH 08/58] Clean up resolveOTLPHeaders --- internal/config/config_helpers.go | 24 ++------------------ internal/config/config_helpers_test.go | 31 -------------------------- 2 files changed, 2 insertions(+), 53 deletions(-) diff --git a/internal/config/config_helpers.go b/internal/config/config_helpers.go index 42f5ca62386..1306aad5643 100644 --- a/internal/config/config_helpers.go +++ b/internal/config/config_helpers.go @@ -190,28 +190,8 @@ func resolveOTLPTraceURL(rawAgentURL *url.URL, otlpTracesEndpoint, otlpEndpoint // OTEL_EXPORTER_OTLP_TRACES_HEADERS. The required Content-Type header for // HTTP protobuf is always included. func resolveOTLPHeaders(otlpTracesHeaders string) map[string]string { - headers := parseOTLPHeaders(otlpTracesHeaders) - headers["Content-Type"] = OTLPContentTypeHeader - return headers -} - -// parseOTLPHeaders parses a comma-separated list of key=value pairs as -// defined by the OTEL_EXPORTER_OTLP_*_HEADERS specification. -func parseOTLPHeaders(s string) map[string]string { headers := make(map[string]string) - if s == "" { - return headers - } - for _, pair := range strings.Split(s, ",") { - pair = strings.TrimSpace(pair) - if pair == "" { - continue - } - k, v, ok := strings.Cut(pair, "=") - if !ok || strings.TrimSpace(k) == "" { - continue - } - headers[strings.TrimSpace(k)] = strings.TrimSpace(v) - } + internal.ForEachStringTag(otlpTracesHeaders, "=", func(k, v string) { headers[k] = v }) + headers["Content-Type"] = OTLPContentTypeHeader return headers } diff --git a/internal/config/config_helpers_test.go b/internal/config/config_helpers_test.go index fad1b9feff4..2b8be5e082f 100644 --- a/internal/config/config_helpers_test.go +++ b/internal/config/config_helpers_test.go @@ -103,34 +103,3 @@ func TestResolveOTLPHeaders(t *testing.T) { }) } -func TestParseOTLPHeaders(t *testing.T) { - t.Run("parses comma-separated key=value pairs", func(t *testing.T) { - got := parseOTLPHeaders("api-key=key,other-config=value") - assert.Equal(t, map[string]string{"api-key": "key", "other-config": "value"}, got) - }) - - t.Run("trims whitespace around keys and values", func(t *testing.T) { - got := parseOTLPHeaders(" api-key = key , other = value ") - assert.Equal(t, map[string]string{"api-key": "key", "other": "value"}, got) - }) - - t.Run("skips empty pairs", func(t *testing.T) { - got := parseOTLPHeaders("api-key=key,,other=value,") - assert.Equal(t, map[string]string{"api-key": "key", "other": "value"}, got) - }) - - t.Run("skips pairs without =", func(t *testing.T) { - got := parseOTLPHeaders("valid=yes,invalid,also-valid=ok") - assert.Equal(t, map[string]string{"valid": "yes", "also-valid": "ok"}, got) - }) - - t.Run("empty string returns empty map", func(t *testing.T) { - got := parseOTLPHeaders("") - assert.Empty(t, got) - }) - - t.Run("value can contain equals sign", func(t *testing.T) { - got := parseOTLPHeaders("auth=base64encoded==") - assert.Equal(t, "base64encoded==", got["auth"]) - }) -} From 51f72a27f3271c1458f32884e95989709865fade Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Mon, 23 Mar 2026 11:56:54 -0400 Subject: [PATCH 09/58] cleanup --- ddtrace/tracer/option.go | 4 ++-- ddtrace/tracer/tracer_test.go | 4 ++-- ddtrace/tracer/transport.go | 6 +++--- ddtrace/tracer/transport_bench_test.go | 4 ++-- ddtrace/tracer/transport_test.go | 10 +++++----- internal/config/config.go | 3 +-- internal/config/config_helpers.go | 10 +++------- internal/config/provider/otelenvconfigsource.go | 3 +-- internal/config/provider/otelenvconfigsource_test.go | 7 ------- 9 files changed, 19 insertions(+), 32 deletions(-) diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 98dc1996ae1..7b19db6de1f 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -423,12 +423,12 @@ func newConfig(opts ...StartOption) (*config, error) { } } if c.transport == nil { - agentURL := c.internalConfig.AgentURL().String() + statsURL := c.internalConfig.AgentURL().String() + statsAPIPath headers := datadogHeaders() if c.internalConfig.OTLPExportMode() { headers = c.internalConfig.OTLPHeaders() } - c.transport = newHTTPTransport(c.internalConfig.TraceURL(), agentURL, c.httpClient, headers) + c.transport = newHTTPTransport(c.internalConfig.TraceURL(), statsURL, c.httpClient, headers) } if c.propagator == nil { envKey := "DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH" diff --git a/ddtrace/tracer/tracer_test.go b/ddtrace/tracer/tracer_test.go index 1af26f9197e..03fb97a6c20 100644 --- a/ddtrace/tracer/tracer_test.go +++ b/ddtrace/tracer/tracer_test.go @@ -1239,7 +1239,7 @@ func TestTracerNoDebugStack(t *testing.T) { // newDefaultTransport return a default transport for this tracing client func newDefaultTransport() transport { - return newHTTPTransport(defaultURL+tracesAPIPath, defaultURL, internal.DefaultHTTPClient(defaultHTTPTimeout, true), datadogHeaders()) + return newHTTPTransport(defaultURL+tracesAPIPath, defaultURL+statsAPIPath, internal.DefaultHTTPClient(defaultHTTPTimeout, true), datadogHeaders()) } func TestNewSpan(t *testing.T) { @@ -1358,7 +1358,7 @@ func TestTracerPrioritySampler(t *testing.T) { url := "http://" + srv.Listener.Addr().String() tr, _, flush, stop, err := startTestTracer(t, - withTransport(newHTTPTransport(url+tracesAPIPath, url, internal.DefaultHTTPClient(defaultHTTPTimeout, false), datadogHeaders())), + withTransport(newHTTPTransport(url+tracesAPIPath, url+statsAPIPath, internal.DefaultHTTPClient(defaultHTTPTimeout, false), datadogHeaders())), ) assert.Nil(err) defer stop() diff --git a/ddtrace/tracer/transport.go b/ddtrace/tracer/transport.go index 80e897251c0..b7f7009b487 100644 --- a/ddtrace/tracer/transport.go +++ b/ddtrace/tracer/transport.go @@ -64,14 +64,14 @@ type httpTransport struct { } // newHTTPTransport returns a new Transport implementation that sends traces -// to the given traceURL and stats to the given agentURL, using the provided +// to the given traceURL and stats to the given statsURL, using the provided // *http.Client and headers. The caller is responsible for providing the // appropriate headers (e.g. datadogHeaders() for Datadog mode, or OTLP // headers resolved from config). -func newHTTPTransport(traceURL string, agentURL string, client *http.Client, headers map[string]string) *httpTransport { +func newHTTPTransport(traceURL string, statsURL string, client *http.Client, headers map[string]string) *httpTransport { return &httpTransport{ traceURL: traceURL, - statsURL: fmt.Sprintf("%s%s", agentURL, statsAPIPath), + statsURL: statsURL, client: client, headers: headers, } diff --git a/ddtrace/tracer/transport_bench_test.go b/ddtrace/tracer/transport_bench_test.go index e747ab325e0..4bc2711b921 100644 --- a/ddtrace/tracer/transport_bench_test.go +++ b/ddtrace/tracer/transport_bench_test.go @@ -24,7 +24,7 @@ func BenchmarkHTTPTransportSend(b *testing.B) { })) defer server.Close() - transport := newHTTPTransport(server.URL+tracesAPIPath, server.URL, internal.DefaultHTTPClient(5*time.Second, false), datadogHeaders()) + transport := newHTTPTransport(server.URL+tracesAPIPath, server.URL+statsAPIPath, internal.DefaultHTTPClient(5*time.Second, false), datadogHeaders()) payloadSizes := []struct { name string @@ -69,7 +69,7 @@ func BenchmarkTransportSendConcurrent(b *testing.B) { })) defer server.Close() - transport := newHTTPTransport(server.URL+tracesAPIPath, server.URL, internal.DefaultHTTPClient(5*time.Second, false), datadogHeaders()) + transport := newHTTPTransport(server.URL+tracesAPIPath, server.URL+statsAPIPath, internal.DefaultHTTPClient(5*time.Second, false), datadogHeaders()) concurrencyLevels := []int{1, 2, 4, 8} for _, concurrency := range concurrencyLevels { diff --git a/ddtrace/tracer/transport_test.go b/ddtrace/tracer/transport_test.go index 4b46157a80a..e5b78afc3be 100644 --- a/ddtrace/tracer/transport_test.go +++ b/ddtrace/tracer/transport_test.go @@ -90,7 +90,7 @@ func TestTracesAgentIntegration(t *testing.T) { } for _, tc := range testCases { - transport := newHTTPTransport(defaultURL+tracesAPIPath, defaultURL, internal.DefaultHTTPClient(defaultHTTPTimeout, false), datadogHeaders()) + transport := newHTTPTransport(defaultURL+tracesAPIPath, defaultURL+statsAPIPath, internal.DefaultHTTPClient(defaultHTTPTimeout, false), datadogHeaders()) p, err := encode(tc.payload) assert.NoError(err) body, err := transport.send(p) @@ -167,7 +167,7 @@ func TestTransportResponse(t *testing.T) { w.Write([]byte(tt.body)) })) defer srv.Close() - transport := newHTTPTransport(srv.URL+tracesAPIPath, srv.URL, internal.DefaultHTTPClient(defaultHTTPTimeout, false), datadogHeaders()) + transport := newHTTPTransport(srv.URL+tracesAPIPath, srv.URL+statsAPIPath, internal.DefaultHTTPClient(defaultHTTPTimeout, false), datadogHeaders()) rc, err := transport.send(newPayload(traceProtocolV04)) if tt.err != "" { assert.Equal(tt.err, err.Error()) @@ -207,7 +207,7 @@ func TestTraceCountHeader(t *testing.T) { })) defer srv.Close() for _, tc := range testCases { - transport := newHTTPTransport(srv.URL+tracesAPIPath, srv.URL, internal.DefaultHTTPClient(defaultHTTPTimeout, false), datadogHeaders()) + transport := newHTTPTransport(srv.URL+tracesAPIPath, srv.URL+statsAPIPath, internal.DefaultHTTPClient(defaultHTTPTimeout, false), datadogHeaders()) p, err := encode(tc.payload) assert.NoError(err) _, err = transport.send(p) @@ -249,7 +249,7 @@ func TestCustomTransport(t *testing.T) { c := &http.Client{} crt := wrapRecordingRoundTripper(c) - transport := newHTTPTransport(srv.URL+tracesAPIPath, srv.URL, c, datadogHeaders()) + transport := newHTTPTransport(srv.URL+tracesAPIPath, srv.URL+statsAPIPath, c, datadogHeaders()) p, err := encode(getTestTrace(1, 1)) assert.NoError(err) _, err = transport.send(p) @@ -634,7 +634,7 @@ func TestConcurrentTraceFlushOverUDS(t *testing.T) { udsURL := internal.UnixDataSocketURL(socketPath).String() client := internal.UDSClient(socketPath, 5*time.Second) - transport := newHTTPTransport(udsURL+tracesAPIPath, udsURL, client, datadogHeaders()) + transport := newHTTPTransport(udsURL+tracesAPIPath, udsURL+statsAPIPath, client, datadogHeaders()) const numGoroutines = 20 diff --git a/internal/config/config.go b/internal/config/config.go index fb94aeb3681..831d7bcec59 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -111,13 +111,12 @@ type Config struct { // Only meaningful when otlpExportMode is false. traceProtocol float64 // otlpExportMode indicates traces should be exported via OTLP rather than - // a Datadog protocol. Set when OTEL_TRACES_EXPORTER=otlp. + // a Datadog protocol. otlpExportMode bool // traceURL is the endpoint URL for sending traces. traceURL string // otlpHeaders holds the resolved OTLP trace headers from // OTEL_EXPORTER_OTLP_TRACES_HEADERS plus Content-Type: application/x-protobuf. - // Always populated; the tracer decides whether to use them based on otlpExportMode. otlpHeaders map[string]string } diff --git a/internal/config/config_helpers.go b/internal/config/config_helpers.go index 1306aad5643..686ed822c99 100644 --- a/internal/config/config_helpers.go +++ b/internal/config/config_helpers.go @@ -83,14 +83,10 @@ func validateTraceProtocolVersion(v string) bool { } func resolveTraceProtocol(v string) float64 { - switch v { - case TraceProtocolVersionStringV04: - return TraceProtocolV04 - case TraceProtocolVersionStringV1: + if v == TraceProtocolVersionStringV1 { return TraceProtocolV1 - default: - return TraceProtocolV04 } + return TraceProtocolV04 } // resolveAgentURL computes the final agent URL from the three env-var strings @@ -165,7 +161,7 @@ func resolveTraceURL(otlpMode bool, protocol float64, rawAgentURL *url.URL, otlp return agentHTTPURL.String() + TracesPathV04 } -// resolveOTLPTraceURL resolves the OTLP trace endpoint using the standard +// resolveOTLPTraceURL resolves the OTLP trace endpoint using the following // OTel env-var priority: // 1. OTEL_EXPORTER_OTLP_TRACES_ENDPOINT (full URL) // 2. OTEL_EXPORTER_OTLP_ENDPOINT (base URL) + /v1/traces diff --git a/internal/config/provider/otelenvconfigsource.go b/internal/config/provider/otelenvconfigsource.go index c1b7700804b..ff61cebd1d3 100644 --- a/internal/config/provider/otelenvconfigsource.go +++ b/internal/config/provider/otelenvconfigsource.go @@ -135,7 +135,7 @@ func mapEnabled(ot string) (string, error) { case "none": return "false", nil case "otlp": - return "true", nil + return "true", nil // Handled separately by otlpExportMode default: return "", fmt.Errorf("the following configuration is not supported: OTEL_TRACES_EXPORTER=%v", ot) } @@ -202,4 +202,3 @@ func mapDDTags(ot string) (string, error) { return strings.Join(ddTags, ","), nil } - diff --git a/internal/config/provider/otelenvconfigsource_test.go b/internal/config/provider/otelenvconfigsource_test.go index dd070097da8..c97c45d4a9f 100644 --- a/internal/config/provider/otelenvconfigsource_test.go +++ b/internal/config/provider/otelenvconfigsource_test.go @@ -77,13 +77,6 @@ func TestOtelEnvConfigSource(t *testing.T) { assert.Equal(t, "false", v) }) - t.Run("maps OTEL_TRACES_EXPORTER=otlp to DD_TRACE_ENABLED=true", func(t *testing.T) { - t.Setenv("OTEL_TRACES_EXPORTER", "otlp") - source := &otelEnvConfigSource{} - v := source.get("DD_TRACE_ENABLED") - assert.Equal(t, "true", v) - }) - t.Run("returns empty for invalid OTEL_TRACES_EXPORTER", func(t *testing.T) { telemetryClient := new(telemetrytest.RecordClient) defer telemetry.MockClient(telemetryClient)() From b71eec7f8c1205654fbf355ada754377b36eca0f Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Mon, 23 Mar 2026 12:01:22 -0400 Subject: [PATCH 10/58] more cleanup and remove OTEL_EXPORTER_OTLP_ENDPOINT --- internal/config/config.go | 5 ++-- internal/config/config_helpers.go | 16 +++--------- internal/config/config_helpers_test.go | 35 ++++++++------------------ internal/config/config_test.go | 27 -------------------- 4 files changed, 16 insertions(+), 67 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 831d7bcec59..a43b6e67571 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -164,8 +164,7 @@ func loadConfig() *Config { cfg.traceProtocol = resolveTraceProtocol(p.GetStringWithValidator("DD_TRACE_AGENT_PROTOCOL_VERSION", TraceProtocolVersionStringV04, validateTraceProtocolVersion)) cfg.otlpExportMode = p.GetString("OTEL_TRACES_EXPORTER", "") == "otlp" otlpTracesEndpoint := p.GetString("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") - otlpEndpoint := p.GetString("OTEL_EXPORTER_OTLP_ENDPOINT", "") - cfg.traceURL = resolveTraceURL(cfg.otlpExportMode, cfg.traceProtocol, cfg.agentURL, otlpTracesEndpoint, otlpEndpoint) + cfg.traceURL = resolveTraceURL(cfg.otlpExportMode, cfg.traceProtocol, cfg.agentURL, otlpTracesEndpoint) cfg.otlpHeaders = resolveOTLPHeaders(p.GetString("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "")) // Parse feature flags from DD_TRACE_FEATURES as a set @@ -711,7 +710,7 @@ func (c *Config) SetTraceProtocol(v float64, origin telemetry.Origin) { c.mu.Lock() defer c.mu.Unlock() c.traceProtocol = v - c.traceURL = resolveTraceURL(c.otlpExportMode, v, c.agentURL, "", "") + c.traceURL = resolveTraceURL(c.otlpExportMode, v, c.agentURL, "") configtelemetry.Report("DD_TRACE_AGENT_PROTOCOL_VERSION", v, origin) } diff --git a/internal/config/config_helpers.go b/internal/config/config_helpers.go index 686ed822c99..65c96f3f914 100644 --- a/internal/config/config_helpers.go +++ b/internal/config/config_helpers.go @@ -10,7 +10,6 @@ import ( "net" "net/url" "os" - "strings" "github.com/DataDog/dd-trace-go/v2/internal" "github.com/DataDog/dd-trace-go/v2/internal/log" @@ -147,9 +146,9 @@ func detectUDSURL() *url.URL { // resolveTraceURL computes the full trace endpoint URL. In OTLP export mode // it uses the OTLP endpoint values; otherwise it derives the URL from the // agent URL and Datadog protocol version. -func resolveTraceURL(otlpMode bool, protocol float64, rawAgentURL *url.URL, otlpTracesEndpoint, otlpEndpoint string) string { +func resolveTraceURL(otlpMode bool, protocol float64, rawAgentURL *url.URL, otlpTracesEndpoint string) string { if otlpMode { - return resolveOTLPTraceURL(rawAgentURL, otlpTracesEndpoint, otlpEndpoint) + return resolveOTLPTraceURL(rawAgentURL, otlpTracesEndpoint) } agentHTTPURL := rawAgentURL if rawAgentURL != nil && rawAgentURL.Scheme == URLSchemeUnix { @@ -161,18 +160,11 @@ func resolveTraceURL(otlpMode bool, protocol float64, rawAgentURL *url.URL, otlp return agentHTTPURL.String() + TracesPathV04 } -// resolveOTLPTraceURL resolves the OTLP trace endpoint using the following -// OTel env-var priority: -// 1. OTEL_EXPORTER_OTLP_TRACES_ENDPOINT (full URL) -// 2. OTEL_EXPORTER_OTLP_ENDPOINT (base URL) + /v1/traces -// 3. agentURL host + default OTLP port 4318 + /v1/traces -func resolveOTLPTraceURL(rawAgentURL *url.URL, otlpTracesEndpoint, otlpEndpoint string) string { +// resolveOTLPTraceURL resolves the OTLP trace endpoint from OTEL_EXPORTER_OTLP_TRACES_ENDPOINT if set, else agentURL host + default OTLP port 4318 + /v1/traces +func resolveOTLPTraceURL(rawAgentURL *url.URL, otlpTracesEndpoint string) string { if otlpTracesEndpoint != "" { return otlpTracesEndpoint } - if otlpEndpoint != "" { - return strings.TrimRight(otlpEndpoint, "/") + otlpTracesPath - } host := internal.DefaultAgentHostname if rawAgentURL != nil { if h := rawAgentURL.Hostname(); h != "" { diff --git a/internal/config/config_helpers_test.go b/internal/config/config_helpers_test.go index 2b8be5e082f..267a4c5ed5c 100644 --- a/internal/config/config_helpers_test.go +++ b/internal/config/config_helpers_test.go @@ -17,33 +17,28 @@ func TestResolveTraceURL(t *testing.T) { unixAgent := &url.URL{Scheme: "unix", Path: "/var/run/datadog/apm.socket"} t.Run("v0.4 protocol uses agent URL", func(t *testing.T) { - got := resolveTraceURL(false, TraceProtocolV04, httpAgent, "", "") + got := resolveTraceURL(false, TraceProtocolV04, httpAgent, "") assert.Equal(t, "http://myhost:8126/v0.4/traces", got) }) t.Run("v1 protocol uses agent URL", func(t *testing.T) { - got := resolveTraceURL(false, TraceProtocolV1, httpAgent, "", "") + got := resolveTraceURL(false, TraceProtocolV1, httpAgent, "") assert.Equal(t, "http://myhost:8126/v1.0/traces", got) }) t.Run("v0.4 with unix socket rewrites to HTTP", func(t *testing.T) { - got := resolveTraceURL(false, TraceProtocolV04, unixAgent, "", "") + got := resolveTraceURL(false, TraceProtocolV04, unixAgent, "") assert.Contains(t, got, "/v0.4/traces") assert.Contains(t, got, "http://") }) t.Run("OTLP mode delegates to resolveOTLPTraceURL", func(t *testing.T) { - got := resolveTraceURL(true, TraceProtocolV04, httpAgent, "", "") + got := resolveTraceURL(true, TraceProtocolV04, httpAgent, "") assert.Equal(t, "http://myhost:4318/v1/traces", got) }) t.Run("OTLP mode with traces endpoint set", func(t *testing.T) { - got := resolveTraceURL(true, TraceProtocolV04, httpAgent, "http://collector:4318/v1/traces", "") - assert.Equal(t, "http://collector:4318/v1/traces", got) - }) - - t.Run("OTLP mode with base endpoint set", func(t *testing.T) { - got := resolveTraceURL(true, TraceProtocolV04, httpAgent, "", "http://collector:4318") + got := resolveTraceURL(true, TraceProtocolV04, httpAgent, "http://collector:4318/v1/traces") assert.Equal(t, "http://collector:4318/v1/traces", got) }) } @@ -51,34 +46,24 @@ func TestResolveTraceURL(t *testing.T) { func TestResolveOTLPTraceURL(t *testing.T) { httpAgent := &url.URL{Scheme: "http", Host: "myhost:8126"} - t.Run("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT takes priority", func(t *testing.T) { - got := resolveOTLPTraceURL(httpAgent, "http://traces-collector:4318/v1/traces", "http://base-collector:4318") + t.Run("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT used when set", func(t *testing.T) { + got := resolveOTLPTraceURL(httpAgent, "http://traces-collector:4318/v1/traces") assert.Equal(t, "http://traces-collector:4318/v1/traces", got) }) - t.Run("OTEL_EXPORTER_OTLP_ENDPOINT used as base with path appended", func(t *testing.T) { - got := resolveOTLPTraceURL(httpAgent, "", "http://collector:4318") - assert.Equal(t, "http://collector:4318/v1/traces", got) - }) - - t.Run("OTEL_EXPORTER_OTLP_ENDPOINT trailing slash is trimmed", func(t *testing.T) { - got := resolveOTLPTraceURL(httpAgent, "", "http://collector:4318/") - assert.Equal(t, "http://collector:4318/v1/traces", got) - }) - t.Run("default uses agent host with OTLP port", func(t *testing.T) { - got := resolveOTLPTraceURL(httpAgent, "", "") + got := resolveOTLPTraceURL(httpAgent, "") assert.Equal(t, "http://myhost:4318/v1/traces", got) }) t.Run("default with nil agent URL uses localhost", func(t *testing.T) { - got := resolveOTLPTraceURL(nil, "", "") + got := resolveOTLPTraceURL(nil, "") assert.Equal(t, "http://localhost:4318/v1/traces", got) }) t.Run("default with unix socket agent uses localhost", func(t *testing.T) { unixAgent := &url.URL{Scheme: "unix", Path: "/var/run/datadog/apm.socket"} - got := resolveOTLPTraceURL(unixAgent, "", "") + got := resolveOTLPTraceURL(unixAgent, "") assert.Equal(t, "http://localhost:4318/v1/traces", got) }) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ca102e431ff..69153d052ad 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -499,33 +499,6 @@ func TestTraceURLResolution(t *testing.T) { assert.Equal(t, "http://collector:4318/v1/traces", cfg.TraceURL()) }) - t.Run("OTLP with OTEL_EXPORTER_OTLP_ENDPOINT appends path", func(t *testing.T) { - resetGlobalState() - defer resetGlobalState() - - t.Setenv("OTEL_TRACES_EXPORTER", "otlp") - t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector:4318") - - cfg := Get() - require.NotNil(t, cfg) - - assert.Equal(t, "http://collector:4318/v1/traces", cfg.TraceURL()) - }) - - t.Run("OTLP TRACES_ENDPOINT takes priority over ENDPOINT", func(t *testing.T) { - resetGlobalState() - defer resetGlobalState() - - t.Setenv("OTEL_TRACES_EXPORTER", "otlp") - t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "http://specific:4318/v1/traces") - t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://generic:4318") - - cfg := Get() - require.NotNil(t, cfg) - - assert.Equal(t, "http://specific:4318/v1/traces", cfg.TraceURL()) - }) - t.Run("OTLP uses agent host when no OTLP endpoint configured", func(t *testing.T) { resetGlobalState() defer resetGlobalState() From f48d44f9fc97aa910297c64dd0b980f408287ac9 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Mon, 23 Mar 2026 12:39:47 -0400 Subject: [PATCH 11/58] Store otlpTraceURL instead of traceURL on Config; let tracer resolve traceURL --- ddtrace/tracer/option.go | 25 +++++++++---- internal/config/config.go | 12 +++---- internal/config/config_helpers.go | 17 --------- internal/config/config_helpers_test.go | 32 ----------------- internal/config/config_test.go | 49 ++++---------------------- 5 files changed, 30 insertions(+), 105 deletions(-) diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 7b19db6de1f..d5cd957d58c 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -423,12 +423,9 @@ func newConfig(opts ...StartOption) (*config, error) { } } if c.transport == nil { - statsURL := c.internalConfig.AgentURL().String() + statsAPIPath - headers := datadogHeaders() - if c.internalConfig.OTLPExportMode() { - headers = c.internalConfig.OTLPHeaders() - } - c.transport = newHTTPTransport(c.internalConfig.TraceURL(), statsURL, c.httpClient, headers) + agentURL := c.internalConfig.AgentURL().String() + traceURL, headers := resolveTraceTransport(c.internalConfig) + c.transport = newHTTPTransport(traceURL, agentURL+statsAPIPath, c.httpClient, headers) } if c.propagator == nil { envKey := "DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH" @@ -470,7 +467,7 @@ func newConfig(opts ...StartOption) (*config, error) { if c.internalConfig.TraceProtocol() == traceProtocolV1 && !af.v1ProtocolAvailable { c.internalConfig.SetTraceProtocol(traceProtocolV04, internalconfig.OriginCalculated) if t, ok := c.transport.(*httpTransport); ok { - t.traceURL = c.internalConfig.TraceURL() + t.traceURL = agentURL.String() + tracesAPIPath } } @@ -535,6 +532,20 @@ func apmTracingDisabled(c *config) { c.internalConfig.SetRuntimeMetricsV2Enabled(false, internalconfig.OriginCalculated) } +// resolveTraceTransport returns the trace URL and headers for the transport +// based on whether OTLP export mode is active. +func resolveTraceTransport(cfg *internalconfig.Config) (traceURL string, headers map[string]string) { + if cfg.OTLPExportMode() { + return cfg.OTLPTraceURL(), cfg.OTLPHeaders() + } + agentURL := cfg.AgentURL().String() + traceURL = agentURL + tracesAPIPath + if cfg.TraceProtocol() == traceProtocolV1 { + traceURL = agentURL + tracesAPIPathV1 + } + return traceURL, datadogHeaders() +} + // resolveDogstatsdAddr resolves the Dogstatsd address to use, based on the user-defined // address and the agent-reported port. If the agent reports a port, it will be used // instead of the user-defined address' port. UDS paths are honored regardless of the diff --git a/internal/config/config.go b/internal/config/config.go index a43b6e67571..27b9c3cfcdd 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -113,8 +113,8 @@ type Config struct { // otlpExportMode indicates traces should be exported via OTLP rather than // a Datadog protocol. otlpExportMode bool - // traceURL is the endpoint URL for sending traces. - traceURL string + // otlpTraceURL is the OTLP collector endpoint for traces + otlpTraceURL string // otlpHeaders holds the resolved OTLP trace headers from // OTEL_EXPORTER_OTLP_TRACES_HEADERS plus Content-Type: application/x-protobuf. otlpHeaders map[string]string @@ -163,8 +163,7 @@ func loadConfig() *Config { cfg.logsOTelEnabled = p.GetBool("DD_LOGS_OTEL_ENABLED", false) cfg.traceProtocol = resolveTraceProtocol(p.GetStringWithValidator("DD_TRACE_AGENT_PROTOCOL_VERSION", TraceProtocolVersionStringV04, validateTraceProtocolVersion)) cfg.otlpExportMode = p.GetString("OTEL_TRACES_EXPORTER", "") == "otlp" - otlpTracesEndpoint := p.GetString("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") - cfg.traceURL = resolveTraceURL(cfg.otlpExportMode, cfg.traceProtocol, cfg.agentURL, otlpTracesEndpoint) + cfg.otlpTraceURL = resolveOTLPTraceURL(cfg.agentURL, p.GetString("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "")) cfg.otlpHeaders = resolveOTLPHeaders(p.GetString("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "")) // Parse feature flags from DD_TRACE_FEATURES as a set @@ -710,14 +709,13 @@ func (c *Config) SetTraceProtocol(v float64, origin telemetry.Origin) { c.mu.Lock() defer c.mu.Unlock() c.traceProtocol = v - c.traceURL = resolveTraceURL(c.otlpExportMode, v, c.agentURL, "") configtelemetry.Report("DD_TRACE_AGENT_PROTOCOL_VERSION", v, origin) } -func (c *Config) TraceURL() string { +func (c *Config) OTLPTraceURL() string { c.mu.RLock() defer c.mu.RUnlock() - return c.traceURL + return c.otlpTraceURL } func (c *Config) OTLPExportMode() bool { diff --git a/internal/config/config_helpers.go b/internal/config/config_helpers.go index 65c96f3f914..d700cb1f033 100644 --- a/internal/config/config_helpers.go +++ b/internal/config/config_helpers.go @@ -143,23 +143,6 @@ func detectUDSURL() *url.URL { } } -// resolveTraceURL computes the full trace endpoint URL. In OTLP export mode -// it uses the OTLP endpoint values; otherwise it derives the URL from the -// agent URL and Datadog protocol version. -func resolveTraceURL(otlpMode bool, protocol float64, rawAgentURL *url.URL, otlpTracesEndpoint string) string { - if otlpMode { - return resolveOTLPTraceURL(rawAgentURL, otlpTracesEndpoint) - } - agentHTTPURL := rawAgentURL - if rawAgentURL != nil && rawAgentURL.Scheme == URLSchemeUnix { - agentHTTPURL = internal.UnixDataSocketURL(rawAgentURL.Path) - } - if protocol == TraceProtocolV1 { - return agentHTTPURL.String() + TracesPathV1 - } - return agentHTTPURL.String() + TracesPathV04 -} - // resolveOTLPTraceURL resolves the OTLP trace endpoint from OTEL_EXPORTER_OTLP_TRACES_ENDPOINT if set, else agentURL host + default OTLP port 4318 + /v1/traces func resolveOTLPTraceURL(rawAgentURL *url.URL, otlpTracesEndpoint string) string { if otlpTracesEndpoint != "" { diff --git a/internal/config/config_helpers_test.go b/internal/config/config_helpers_test.go index 267a4c5ed5c..278caaf3f0d 100644 --- a/internal/config/config_helpers_test.go +++ b/internal/config/config_helpers_test.go @@ -12,37 +12,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestResolveTraceURL(t *testing.T) { - httpAgent := &url.URL{Scheme: "http", Host: "myhost:8126"} - unixAgent := &url.URL{Scheme: "unix", Path: "/var/run/datadog/apm.socket"} - - t.Run("v0.4 protocol uses agent URL", func(t *testing.T) { - got := resolveTraceURL(false, TraceProtocolV04, httpAgent, "") - assert.Equal(t, "http://myhost:8126/v0.4/traces", got) - }) - - t.Run("v1 protocol uses agent URL", func(t *testing.T) { - got := resolveTraceURL(false, TraceProtocolV1, httpAgent, "") - assert.Equal(t, "http://myhost:8126/v1.0/traces", got) - }) - - t.Run("v0.4 with unix socket rewrites to HTTP", func(t *testing.T) { - got := resolveTraceURL(false, TraceProtocolV04, unixAgent, "") - assert.Contains(t, got, "/v0.4/traces") - assert.Contains(t, got, "http://") - }) - - t.Run("OTLP mode delegates to resolveOTLPTraceURL", func(t *testing.T) { - got := resolveTraceURL(true, TraceProtocolV04, httpAgent, "") - assert.Equal(t, "http://myhost:4318/v1/traces", got) - }) - - t.Run("OTLP mode with traces endpoint set", func(t *testing.T) { - got := resolveTraceURL(true, TraceProtocolV04, httpAgent, "http://collector:4318/v1/traces") - assert.Equal(t, "http://collector:4318/v1/traces", got) - }) -} - func TestResolveOTLPTraceURL(t *testing.T) { httpAgent := &url.URL{Scheme: "http", Host: "myhost:8126"} @@ -87,4 +56,3 @@ func TestResolveOTLPHeaders(t *testing.T) { assert.Equal(t, OTLPContentTypeHeader, got["Content-Type"]) }) } - diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 69153d052ad..dae29ade124 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -461,55 +461,39 @@ func TestSetServiceMappingReportsFullList(t *testing.T) { assert.Equal(t, []string{"a:3", "b:2"}, parts) } -func TestTraceURLResolution(t *testing.T) { - t.Run("default protocol resolves traceURL from agent URL", func(t *testing.T) { +func TestOTLPTraceURLResolution(t *testing.T) { + t.Run("default OTLP port from agent host", func(t *testing.T) { resetGlobalState() defer resetGlobalState() cfg := Get() require.NotNil(t, cfg) - traceURL := cfg.TraceURL() - assert.Contains(t, traceURL, "/v0.4/traces") + assert.Contains(t, cfg.OTLPTraceURL(), ":4318/v1/traces") }) - t.Run("OTLP export mode resolves traceURL with default OTLP port", func(t *testing.T) { + t.Run("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT overrides", func(t *testing.T) { resetGlobalState() defer resetGlobalState() - t.Setenv("OTEL_TRACES_EXPORTER", "otlp") - - cfg := Get() - require.NotNil(t, cfg) - - assert.True(t, cfg.OTLPExportMode()) - assert.Contains(t, cfg.TraceURL(), ":4318/v1/traces") - }) - - t.Run("OTLP with OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", func(t *testing.T) { - resetGlobalState() - defer resetGlobalState() - - t.Setenv("OTEL_TRACES_EXPORTER", "otlp") t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "http://collector:4318/v1/traces") cfg := Get() require.NotNil(t, cfg) - assert.Equal(t, "http://collector:4318/v1/traces", cfg.TraceURL()) + assert.Equal(t, "http://collector:4318/v1/traces", cfg.OTLPTraceURL()) }) - t.Run("OTLP uses agent host when no OTLP endpoint configured", func(t *testing.T) { + t.Run("uses agent host when no OTLP endpoint configured", func(t *testing.T) { resetGlobalState() defer resetGlobalState() - t.Setenv("OTEL_TRACES_EXPORTER", "otlp") t.Setenv("DD_AGENT_HOST", "custom-agent") cfg := Get() require.NotNil(t, cfg) - assert.Equal(t, "http://custom-agent:4318/v1/traces", cfg.TraceURL()) + assert.Equal(t, "http://custom-agent:4318/v1/traces", cfg.OTLPTraceURL()) }) } @@ -544,25 +528,6 @@ func TestOTLPHeaders(t *testing.T) { } -func TestSetTraceProtocolRecomputesTraceURL(t *testing.T) { - resetGlobalState() - defer resetGlobalState() - - cfg := Get() - require.NotNil(t, cfg) - - // Default should be v0.4 - assert.Contains(t, cfg.TraceURL(), "/v0.4/traces") - - // Switch to v1 - cfg.SetTraceProtocol(TraceProtocolV1, telemetry.OriginCode) - assert.Contains(t, cfg.TraceURL(), "/v1.0/traces") - - // Switch back to v0.4 - cfg.SetTraceProtocol(TraceProtocolV04, telemetry.OriginCode) - assert.Contains(t, cfg.TraceURL(), "/v0.4/traces") -} - func TestOTLPExportMode(t *testing.T) { t.Run("disabled by default", func(t *testing.T) { resetGlobalState() From f4360b475cf9ec10d0363a0e10e9234a1ea815ce Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Mon, 23 Mar 2026 14:44:47 -0400 Subject: [PATCH 12/58] Make DD_TRACE_AGENT_PROTOCOL_VERSION override OTEL_TRACES_EXPORTER --- internal/config/config.go | 4 ++++ internal/config/config_test.go | 20 +++++++++++++++++--- internal/config/provider/provider.go | 13 +++++++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 27b9c3cfcdd..91f49a73361 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -163,6 +163,10 @@ func loadConfig() *Config { cfg.logsOTelEnabled = p.GetBool("DD_LOGS_OTEL_ENABLED", false) cfg.traceProtocol = resolveTraceProtocol(p.GetStringWithValidator("DD_TRACE_AGENT_PROTOCOL_VERSION", TraceProtocolVersionStringV04, validateTraceProtocolVersion)) cfg.otlpExportMode = p.GetString("OTEL_TRACES_EXPORTER", "") == "otlp" + // DD_TRACE_AGENT_PROTOCOL_VERSION overrides OTEL_TRACES_EXPORTER + if p.IsSet("DD_TRACE_AGENT_PROTOCOL_VERSION") { + cfg.otlpExportMode = false + } cfg.otlpTraceURL = resolveOTLPTraceURL(cfg.agentURL, p.GetString("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "")) cfg.otlpHeaders = resolveOTLPHeaders(p.GetString("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "")) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index dae29ade124..d2921bc3ab6 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -575,7 +575,7 @@ func TestOTLPExportMode(t *testing.T) { assert.False(t, cfg.OTLPExportMode()) }) - t.Run("independent of traceProtocol", func(t *testing.T) { + t.Run("DD_TRACE_AGENT_PROTOCOL_VERSION overrides OTEL_TRACES_EXPORTER", func(t *testing.T) { resetGlobalState() defer resetGlobalState() @@ -585,8 +585,22 @@ func TestOTLPExportMode(t *testing.T) { cfg := Get() require.NotNil(t, cfg) - assert.True(t, cfg.OTLPExportMode(), "otlpExportMode should be true regardless of traceProtocol") - assert.Equal(t, TraceProtocolV1, cfg.TraceProtocol(), "traceProtocol should reflect DD_TRACE_AGENT_PROTOCOL_VERSION") + assert.False(t, cfg.OTLPExportMode(), "otlpExportMode should be false when DD_TRACE_AGENT_PROTOCOL_VERSION is explicitly set") + assert.Equal(t, TraceProtocolV1, cfg.TraceProtocol()) + }) + + t.Run("DD_TRACE_AGENT_PROTOCOL_VERSION=0.4 still overrides OTEL_TRACES_EXPORTER", func(t *testing.T) { + resetGlobalState() + defer resetGlobalState() + + t.Setenv("OTEL_TRACES_EXPORTER", "otlp") + t.Setenv("DD_TRACE_AGENT_PROTOCOL_VERSION", "0.4") + + cfg := Get() + require.NotNil(t, cfg) + + assert.False(t, cfg.OTLPExportMode(), "otlpExportMode should be false when DD_TRACE_AGENT_PROTOCOL_VERSION is explicitly set, even to the default value") + assert.Equal(t, TraceProtocolV04, cfg.TraceProtocol()) }) t.Run("SetOTLPExportMode toggles mode", func(t *testing.T) { diff --git a/internal/config/provider/provider.go b/internal/config/provider/provider.go index 0c550d0da95..a876c19383b 100644 --- a/internal/config/provider/provider.go +++ b/internal/config/provider/provider.go @@ -154,6 +154,19 @@ func (p *Provider) GetFloatWithValidator(key string, def float64, validate func( }) } +// / IsSet returns true if any configuration source provides a non-empty value for the key. +// +// TODO: populate an isSet field on the Provider at the time of iterating over +// sources instead of re-querying them here. +func (p *Provider) IsSet(key string) bool { + for _, source := range p.sources { + if source.get(key) != "" { + return true + } + } + return false +} + // normalizeKey normalizes the key to a valid environment variable name. func normalizeKey(key string) string { if strings.HasPrefix(key, "DD_") || strings.HasPrefix(key, "OTEL_") { From a88b7b2edaeb43eecdc847ca2eb01b889b2acc17 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Tue, 24 Mar 2026 14:06:24 -0400 Subject: [PATCH 13/58] Use GetMap to resolve OTEL_EXPORTER_OTLP_TRACES_HEADERS and change config registry to reflect this --- internal/config/config.go | 8 +++++--- internal/config/config_helpers.go | 12 ++++++------ internal/config/config_helpers_test.go | 20 -------------------- internal/config/config_test.go | 2 ++ internal/config/provider/provider.go | 11 ++++++----- internal/config/provider/provider_test.go | 7 ++++--- internal/env/supported_configurations.json | 4 ++-- 7 files changed, 25 insertions(+), 39 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 944f3a57104..47144462ad6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -140,14 +140,14 @@ func loadConfig() *Config { cfg.serviceName = p.GetString("DD_SERVICE", "") cfg.version = p.GetString("DD_VERSION", "") cfg.env = p.GetString("DD_ENV", "") - cfg.serviceMappings = p.GetMap("DD_SERVICE_MAPPING", nil) + cfg.serviceMappings = p.GetMap("DD_SERVICE_MAPPING", nil, internal.DDTagsDelimiter) cfg.runtimeMetrics = p.GetBool("DD_RUNTIME_METRICS_ENABLED", false) cfg.runtimeMetricsV2 = p.GetBool("DD_RUNTIME_METRICS_V2_ENABLED", true) cfg.profilerHotspots = p.GetBool("DD_PROFILING_CODE_HOTSPOTS_COLLECTION_ENABLED", true) cfg.profilerEndpoints = p.GetBool("DD_PROFILING_ENDPOINT_COLLECTION_ENABLED", true) cfg.spanAttributeSchemaVersion = p.GetInt("DD_TRACE_SPAN_ATTRIBUTE_SCHEMA", 0) cfg.peerServiceDefaultsEnabled = p.GetBool("DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED", false) - cfg.peerServiceMappings = p.GetMap("DD_TRACE_PEER_SERVICE_MAPPING", nil) + cfg.peerServiceMappings = p.GetMap("DD_TRACE_PEER_SERVICE_MAPPING", nil, internal.DDTagsDelimiter) cfg.debugAbandonedSpans = p.GetBool("DD_TRACE_DEBUG_ABANDONED_SPANS", false) cfg.spanTimeout = p.GetDuration("DD_TRACE_ABANDONED_SPAN_TIMEOUT", 10*time.Minute) cfg.partialFlushMinSpans = p.GetIntWithValidator("DD_TRACE_PARTIAL_FLUSH_MIN_SPANS", 1000, validatePartialFlushMinSpans) @@ -170,7 +170,7 @@ func loadConfig() *Config { cfg.otlpExportMode = false } cfg.otlpTraceURL = resolveOTLPTraceURL(cfg.agentURL, p.GetString("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "")) - cfg.otlpHeaders = resolveOTLPHeaders(p.GetString("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "")) + cfg.otlpHeaders = buildOTLPHeaders(p.GetMap("OTEL_EXPORTER_OTLP_TRACES_HEADERS", nil, internal.OtelTagsDelimeter)) // Parse feature flags from DD_TRACE_FEATURES as a set cfg.featureFlags = make(map[string]struct{}) @@ -737,6 +737,8 @@ func (c *Config) SetOTLPExportMode(v bool, origin telemetry.Origin) { configtelemetry.Report("OTEL_TRACES_EXPORTER", v, origin) } +// OTLPHeaders returns a copy of the OTLP headers map. If no headers are set, returns nil. +// Safe to return the full map because it is not called in hot paths. func (c *Config) OTLPHeaders() map[string]string { c.mu.RLock() defer c.mu.RUnlock() diff --git a/internal/config/config_helpers.go b/internal/config/config_helpers.go index d700cb1f033..0b4c9469f10 100644 --- a/internal/config/config_helpers.go +++ b/internal/config/config_helpers.go @@ -157,12 +157,12 @@ func resolveOTLPTraceURL(rawAgentURL *url.URL, otlpTracesEndpoint string) string return fmt.Sprintf("http://%s:%s%s", host, otlpDefaultPort, otlpTracesPath) } -// resolveOTLPHeaders builds the OTLP trace headers map from -// OTEL_EXPORTER_OTLP_TRACES_HEADERS. The required Content-Type header for -// HTTP protobuf is always included. -func resolveOTLPHeaders(otlpTracesHeaders string) map[string]string { - headers := make(map[string]string) - internal.ForEachStringTag(otlpTracesHeaders, "=", func(k, v string) { headers[k] = v }) +// buildOTLPHeaders builds the OTLP headers map from the provided map. +// It adds the Content-Type header if not present. +func buildOTLPHeaders(headers map[string]string) map[string]string { + if headers == nil { + headers = make(map[string]string) + } headers["Content-Type"] = OTLPContentTypeHeader return headers } diff --git a/internal/config/config_helpers_test.go b/internal/config/config_helpers_test.go index 278caaf3f0d..8f43dde8786 100644 --- a/internal/config/config_helpers_test.go +++ b/internal/config/config_helpers_test.go @@ -36,23 +36,3 @@ func TestResolveOTLPTraceURL(t *testing.T) { assert.Equal(t, "http://localhost:4318/v1/traces", got) }) } - -func TestResolveOTLPHeaders(t *testing.T) { - t.Run("always includes Content-Type for protobuf", func(t *testing.T) { - got := resolveOTLPHeaders("") - assert.Equal(t, OTLPContentTypeHeader, got["Content-Type"]) - }) - - t.Run("user headers plus Content-Type", func(t *testing.T) { - got := resolveOTLPHeaders("api-key=key,other=value") - assert.Equal(t, "key", got["api-key"]) - assert.Equal(t, "value", got["other"]) - assert.Equal(t, OTLPContentTypeHeader, got["Content-Type"]) - }) - - t.Run("empty TRACES_HEADERS still gets Content-Type", func(t *testing.T) { - got := resolveOTLPHeaders("") - assert.Len(t, got, 1) - assert.Equal(t, OTLPContentTypeHeader, got["Content-Type"]) - }) -} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index d2921bc3ab6..f1f667e6f09 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -6,6 +6,7 @@ package config import ( + "fmt" "net/url" "reflect" "sort" @@ -521,6 +522,7 @@ func TestOTLPHeaders(t *testing.T) { require.NotNil(t, cfg) headers := cfg.OTLPHeaders() + fmt.Println(t, headers) assert.Equal(t, "secret", headers["api-key"]) assert.Equal(t, "value", headers["x-custom"]) assert.Equal(t, OTLPContentTypeHeader, headers["Content-Type"]) diff --git a/internal/config/provider/provider.go b/internal/config/provider/provider.go index a876c19383b..a7429fcd448 100644 --- a/internal/config/provider/provider.go +++ b/internal/config/provider/provider.go @@ -120,9 +120,9 @@ func (p *Provider) GetIntWithValidator(key string, def int, validate func(int) b }) } -func (p *Provider) GetMap(key string, def map[string]string) map[string]string { +func (p *Provider) GetMap(key string, def map[string]string, delimiter string) map[string]string { return get(p, key, def, func(v string) (map[string]string, bool) { - m := parseMapString(v) + m := parseMapString(v, delimiter) return m, len(m) > 0 }) } @@ -175,10 +175,11 @@ func normalizeKey(key string) string { return "DD_" + strings.ToUpper(key) } -// parseMapString parses a string containing key:value pairs separated by comma or space. -func parseMapString(str string) map[string]string { +// parseMapString parses a string containing key-value pairs separated by comma or space. +// It prioritizes the Datadog delimiter (:) over the OTel delimiter (=) +func parseMapString(str string, delimiter string) map[string]string { result := make(map[string]string) - internal.ForEachStringTag(str, internal.DDTagsDelimiter, func(key, val string) { + internal.ForEachStringTag(str, delimiter, func(key, val string) { result[key] = val }) return result diff --git a/internal/config/provider/provider_test.go b/internal/config/provider/provider_test.go index 97e2751d434..221573df256 100644 --- a/internal/config/provider/provider_test.go +++ b/internal/config/provider/provider_test.go @@ -15,6 +15,7 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/DataDog/dd-trace-go/v2/internal" "github.com/DataDog/dd-trace-go/v2/internal/telemetry" "github.com/DataDog/dd-trace-go/v2/internal/telemetry/telemetrytest" ) @@ -361,7 +362,7 @@ func TestProviderTelemetryRegistration(t *testing.T) { _ = p.GetInt("DD_TRACE_PARTIAL_FLUSH_MIN_SPANS", 0) _ = p.GetFloat("DD_TRACE_SAMPLE_RATE", 0.0) _ = p.GetString("DD_TRACE_AGENT_URL", "") - _ = p.GetMap("DD_SERVICE_MAPPING", nil) + _ = p.GetMap("DD_SERVICE_MAPPING", nil, internal.DDTagsDelimiter) _ = p.GetDuration("DD_TRACE_ABANDONED_SPAN_TIMEOUT", 0) telemetryClient.AssertCalled(t, "RegisterAppConfigs", mock.MatchedBy(matchConfig("DD_SERVICE", "service", telemetry.OriginEnvVar, telemetry.EmptyID))) @@ -400,7 +401,7 @@ apm_configuration_default: _ = p.GetInt("DD_TRACE_PARTIAL_FLUSH_MIN_SPANS", 0) _ = p.GetFloat("DD_TRACE_SAMPLE_RATE", 0.0) _ = p.GetString("DD_TRACE_AGENT_URL", "") - _ = p.GetMap("DD_SERVICE_MAPPING", nil) + _ = p.GetMap("DD_SERVICE_MAPPING", nil, internal.DDTagsDelimiter) _ = p.GetDuration("DD_TRACE_ABANDONED_SPAN_TIMEOUT", 0) telemetryClient.AssertCalled(t, "RegisterAppConfigs", mock.MatchedBy(matchConfig("DD_SERVICE", "svc", telemetry.OriginLocalStableConfig, "123"))) @@ -487,7 +488,7 @@ apm_configuration_default: assert.Equal(t, intDef, p.GetInt(intKey, intDef)) assert.Equal(t, floatDef, p.GetFloat(floatKey, floatDef)) assert.Equal(t, durDef, p.GetDuration(durKey, durDef)) - assert.Equal(t, mapDef, p.GetMap(mapKey, mapDef)) + assert.Equal(t, mapDef, p.GetMap(mapKey, mapDef, internal.DDTagsDelimiter)) telemetryClient.AssertExpectations(t) }) diff --git a/internal/env/supported_configurations.json b/internal/env/supported_configurations.json index b9170ffded1..ac27f951d0f 100644 --- a/internal/env/supported_configurations.json +++ b/internal/env/supported_configurations.json @@ -1676,8 +1676,8 @@ "OTEL_EXPORTER_OTLP_TRACES_HEADERS": [ { "implementation": "B", - "type": "string", - "default": "" + "type": "map", + "default": null } ], "OTEL_LOGS_EXPORTER": [ From d4759207ba3cca7b9fed0aa0308ffe446d4b9fcb Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Tue, 24 Mar 2026 14:06:38 -0400 Subject: [PATCH 14/58] give OTEL_EXPORTER_OTLP_TRACES_ENDPOINT default value of null --- internal/env/supported_configurations.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/env/supported_configurations.json b/internal/env/supported_configurations.json index ac27f951d0f..789c5b7d45c 100644 --- a/internal/env/supported_configurations.json +++ b/internal/env/supported_configurations.json @@ -1670,7 +1670,7 @@ { "implementation": "B", "type": "string", - "default": "" + "default": null } ], "OTEL_EXPORTER_OTLP_TRACES_HEADERS": [ From af909c308281281440f17de4038182c0b0f1839e Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Tue, 24 Mar 2026 15:11:35 -0400 Subject: [PATCH 15/58] flesh out traceWriterOtlp --- ddtrace/tracer/otlp_writer.go | 80 +++++++++++++++++++++++--- ddtrace/tracer/payload_otlp_convert.go | 19 ++++++ 2 files changed, 91 insertions(+), 8 deletions(-) create mode 100644 ddtrace/tracer/payload_otlp_convert.go diff --git a/ddtrace/tracer/otlp_writer.go b/ddtrace/tracer/otlp_writer.go index c196372b8d7..86f1acc4bb6 100644 --- a/ddtrace/tracer/otlp_writer.go +++ b/ddtrace/tracer/otlp_writer.go @@ -5,18 +5,82 @@ package tracer +import ( + "slices" + "sync" + + "github.com/DataDog/dd-trace-go/v2/internal/locking" + "github.com/DataDog/dd-trace-go/v2/internal/log" + otlpcommon "go.opentelemetry.io/proto/otlp/common/v1" + otlpresource "go.opentelemetry.io/proto/otlp/resource/v1" + otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" + "google.golang.org/protobuf/proto" +) + var _ traceWriter = (*otlpTraceWriter)(nil) -// otlpTraceWriter is a no-op placeholder for the OTLP export trace writer. -// TODO: implement OTLP protobuf encoding and export to OTel Collector. -type otlpTraceWriter struct{} +type otlpTraceWriter struct { + config *config + mu locking.Mutex + resource *otlpresource.Resource + scope *otlpcommon.InstrumentationScope + spans []*otlptrace.Span // +checklocks:mu + climit chan struct{} + wg sync.WaitGroup +} -func newOTLPTraceWriter(_ *config) *otlpTraceWriter { - return &otlpTraceWriter{} +func newOTLPTraceWriter(c *config) *otlpTraceWriter { + return &otlpTraceWriter{ + config: c, + resource: buildResource(c), + scope: &otlpcommon.InstrumentationScope{Name: "dd-trace-go"}, + spans: make([]*otlptrace.Span, 0), + climit: make(chan struct{}, concurrentConnectionLimit), + wg: sync.WaitGroup{}, + } } -func (w *otlpTraceWriter) add(_ []*Span) {} +func (w *otlpTraceWriter) add(spanList []*Span) { + w.mu.Lock() + defer w.mu.Unlock() + w.spans = slices.Grow(w.spans, len(spanList)) + for _, span := range spanList { + w.spans = append(w.spans, convertSpan(span)) + } +} -func (w *otlpTraceWriter) flush() {} +func (w *otlpTraceWriter) flush() { + w.mu.Lock() + defer w.mu.Unlock() + tracesData := &otlptrace.TracesData{ + ResourceSpans: []*otlptrace.ResourceSpans{ + { + Resource: w.resource, + ScopeSpans: []*otlptrace.ScopeSpans{ + { + Scope: w.scope, + Spans: w.spans, + }, + }, + }, + }, + } + _, err := proto.Marshal(tracesData) + if err != nil { + log.Error("Error marshalling OTLP traces data: %s", err.Error()) + return + } + // w.climit <- struct{}{} + // w.wg.Add(1) + // go func() { + // defer w.wg.Done() + // <-w.climit + // w.config.transport.send(b) + // }() +} -func (w *otlpTraceWriter) stop() {} +func (w *otlpTraceWriter) stop() { + // TODO: agentTraceWriter reports datadog.tracer.flush_triggered to its statsd client here; do we want to do the same here? + w.flush() + w.wg.Wait() +} diff --git a/ddtrace/tracer/payload_otlp_convert.go b/ddtrace/tracer/payload_otlp_convert.go new file mode 100644 index 00000000000..91cd60deb1b --- /dev/null +++ b/ddtrace/tracer/payload_otlp_convert.go @@ -0,0 +1,19 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016 Datadog, Inc. + +package tracer + +import ( + otlpresource "go.opentelemetry.io/proto/otlp/resource/v1" + otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" +) + +func buildResource(c *config) *otlpresource.Resource { + return nil +} + +func convertSpan(s *Span) *otlptrace.Span { + return nil +} From 92460841895f89fcf564efce6657c0fd2ea6a0dd Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Tue, 24 Mar 2026 16:05:01 -0400 Subject: [PATCH 16/58] add new otlpSender for transporting otlp traces specifically; build out ddspan -> otlpspan conversion --- ddtrace/tracer/otlp_writer.go | 91 +++++++--- ddtrace/tracer/payload_otlp_convert.go | 223 ++++++++++++++++++++++++- ddtrace/tracer/transport.go | 32 +++- 3 files changed, 318 insertions(+), 28 deletions(-) diff --git a/ddtrace/tracer/otlp_writer.go b/ddtrace/tracer/otlp_writer.go index 86f1acc4bb6..3227a6b830e 100644 --- a/ddtrace/tracer/otlp_writer.go +++ b/ddtrace/tracer/otlp_writer.go @@ -6,8 +6,11 @@ package tracer import ( + "fmt" + "io" "slices" "sync" + "time" "github.com/DataDog/dd-trace-go/v2/internal/locking" "github.com/DataDog/dd-trace-go/v2/internal/log" @@ -21,6 +24,7 @@ var _ traceWriter = (*otlpTraceWriter)(nil) type otlpTraceWriter struct { config *config + sender otlpSender mu locking.Mutex resource *otlpresource.Resource scope *otlpcommon.InstrumentationScope @@ -29,10 +33,24 @@ type otlpTraceWriter struct { wg sync.WaitGroup } +// noopOTLPSender is a fallback sender that drops all payloads. Used when the +// configured transport does not implement otlpSender. +type noopOTLPSender struct{} + +func (noopOTLPSender) sendOTLP([]byte) (io.ReadCloser, error) { + return nil, fmt.Errorf("OTLP sending is not supported by the configured transport") +} + func newOTLPTraceWriter(c *config) *otlpTraceWriter { + sender, ok := c.transport.(otlpSender) + if !ok { + log.Error("OTLP trace writer: transport %T does not implement otlpSender; traces will be dropped", c.transport) + sender = noopOTLPSender{} + } return &otlpTraceWriter{ config: c, - resource: buildResource(c), + sender: sender, + resource: buildResource(c.internalConfig), scope: &otlpcommon.InstrumentationScope{Name: "dd-trace-go"}, spans: make([]*otlptrace.Span, 0), climit: make(chan struct{}, concurrentConnectionLimit), @@ -51,32 +69,57 @@ func (w *otlpTraceWriter) add(spanList []*Span) { func (w *otlpTraceWriter) flush() { w.mu.Lock() - defer w.mu.Unlock() - tracesData := &otlptrace.TracesData{ - ResourceSpans: []*otlptrace.ResourceSpans{ - { - Resource: w.resource, - ScopeSpans: []*otlptrace.ScopeSpans{ - { - Scope: w.scope, - Spans: w.spans, + if len(w.spans) == 0 { + w.mu.Unlock() + return + } + readySpans := w.spans + w.spans = make([]*otlptrace.Span, 0) + w.mu.Unlock() + + w.climit <- struct{}{} + w.wg.Add(1) + go func() { + defer func() { + <-w.climit + w.wg.Done() + }() + + spanCount := len(readySpans) + tracesData := &otlptrace.TracesData{ + ResourceSpans: []*otlptrace.ResourceSpans{ + { + Resource: w.resource, + ScopeSpans: []*otlptrace.ScopeSpans{ + { + Scope: w.scope, + Spans: readySpans, + }, }, }, }, - }, - } - _, err := proto.Marshal(tracesData) - if err != nil { - log.Error("Error marshalling OTLP traces data: %s", err.Error()) - return - } - // w.climit <- struct{}{} - // w.wg.Add(1) - // go func() { - // defer w.wg.Done() - // <-w.climit - // w.config.transport.send(b) - // }() + } + b, err := proto.Marshal(tracesData) + readySpans = nil + tracesData = nil + if err != nil { + log.Error("Error marshalling OTLP traces data: %s", err.Error()) + return + } + + var sendErr error + for attempt := 0; attempt <= w.config.sendRetries; attempt++ { + log.Debug("OTLP: attempt %d to send payload: %d bytes, %d spans", attempt+1, len(b), spanCount) + _, sendErr = w.sender.sendOTLP(b) + if sendErr == nil { + log.Debug("OTLP: sent traces after %d attempts", attempt+1) + return + } + log.Error("OTLP: failure sending traces (attempt %d of %d): %v", attempt+1, w.config.sendRetries+1, sendErr) + time.Sleep(w.config.internalConfig.RetryInterval()) + } + log.Error("OTLP: lost %d spans: %v", spanCount, sendErr) + }() } func (w *otlpTraceWriter) stop() { diff --git a/ddtrace/tracer/payload_otlp_convert.go b/ddtrace/tracer/payload_otlp_convert.go index 91cd60deb1b..631c9f5389f 100644 --- a/ddtrace/tracer/payload_otlp_convert.go +++ b/ddtrace/tracer/payload_otlp_convert.go @@ -6,14 +6,231 @@ package tracer import ( + "encoding/binary" + + internalconfig "github.com/DataDog/dd-trace-go/v2/internal/config" + otlpcommon "go.opentelemetry.io/proto/otlp/common/v1" otlpresource "go.opentelemetry.io/proto/otlp/resource/v1" otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" + + "github.com/DataDog/dd-trace-go/v2/ddtrace/ext" ) -func buildResource(c *config) *otlpresource.Resource { - return nil +// ----------------------------------------------------------------------------- +// Resource construction +// ----------------------------------------------------------------------------- + +// buildResource constructs the OTLP Resource from resolved tracer configuration. +// If cfg is nil, an empty resource is returned. +func buildResource(cfg *internalconfig.Config) *otlpresource.Resource { + if cfg == nil { + return &otlpresource.Resource{} + } + attrs := []*otlpcommon.KeyValue{ + otlpKeyValue("service.name", otlpStringValue(cfg.ServiceName())), + otlpKeyValue("telemetry.sdk.language", otlpStringValue("go")), + otlpKeyValue("telemetry.sdk.name", otlpStringValue("datadog")), + otlpKeyValue("telemetry.sdk.version", otlpStringValue(cfg.Version())), + } + if v := cfg.Env(); v != "" { + attrs = append(attrs, otlpKeyValue("deployment.environment", otlpStringValue(v))) + } + if v := cfg.Version(); v != "" { + attrs = append(attrs, otlpKeyValue("service.version", otlpStringValue(v))) + } + return &otlpresource.Resource{Attributes: attrs} } +// ----------------------------------------------------------------------------- +// Span conversion (DD Span → OTLP Span and related types) +// ----------------------------------------------------------------------------- + +// +checklocksignore — Post-finish: reads finished span fields during payload encoding. func convertSpan(s *Span) *otlptrace.Span { - return nil + return &otlptrace.Span{ + TraceId: convertTraceID(s.context.traceID.Upper(), s.context.traceID.Lower()), + SpanId: convertSpanID(s.spanID), + Name: s.resource, + Kind: convertSpanKind(getSpanKind(s)), + StartTimeUnixNano: uint64(s.start), + EndTimeUnixNano: uint64(s.start + s.duration), + Attributes: convertSpanAttributes(s), + Events: convertEvents(s), + Links: convertSpanLinks(s.spanLinks), + Status: convertSpanStatus(s), + } +} + +// +checklocksignore — Post-finish: reads finished span fields during payload encoding. +func convertSpanStatus(s *Span) *otlptrace.Status { + status := &otlptrace.Status{ + Code: otlptrace.Status_STATUS_CODE_UNSET, + Message: s.meta[ext.ErrorMsg], + } + if s.error == 1 { + status.Code = otlptrace.Status_STATUS_CODE_ERROR + } + return status +} + +func convertSpanLinks(links []SpanLink) []*otlptrace.Span_Link { + if len(links) == 0 { + return nil + } + otlpLinks := make([]*otlptrace.Span_Link, 0) + for _, link := range links { + otlpLinks = append(otlpLinks, &otlptrace.Span_Link{ + TraceId: convertTraceID(link.TraceIDHigh, link.TraceID), + SpanId: convertSpanID(link.SpanID), + Attributes: convertMapToOTLPAttributesString(link.Attributes), + TraceState: link.Tracestate, + Flags: link.Flags, + }) + } + return otlpLinks +} + +// +checklocksignore — Post-finish: reads finished span fields during payload encoding. +func convertEvents(s *Span) []*otlptrace.Span_Event { + if len(s.spanEvents) == 0 { + return nil + } + events := make([]*otlptrace.Span_Event, 0) + for _, event := range s.spanEvents { + events = append(events, &otlptrace.Span_Event{ + Name: event.Name, + TimeUnixNano: uint64(event.TimeUnixNano), + Attributes: convertEventAttributes(event.Attributes), + }) + } + return events +} + +func convertTraceID(high, low uint64) []byte { + b := make([]byte, 16) + binary.BigEndian.PutUint64(b[:8], high) + binary.BigEndian.PutUint64(b[8:], low) + return b +} + +func convertSpanID(spanID uint64) []byte { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, spanID) + return b +} + +func convertSpanKind(spanKind string) otlptrace.Span_SpanKind { + switch spanKind { + case ext.SpanKindInternal: + return otlptrace.Span_SPAN_KIND_INTERNAL + case ext.SpanKindServer: + return otlptrace.Span_SPAN_KIND_SERVER + case ext.SpanKindClient: + return otlptrace.Span_SPAN_KIND_CLIENT + case ext.SpanKindProducer: + return otlptrace.Span_SPAN_KIND_PRODUCER + case ext.SpanKindConsumer: + return otlptrace.Span_SPAN_KIND_CONSUMER + default: + return otlptrace.Span_SPAN_KIND_INTERNAL + } +} + +// +checklocksignore — Post-finish: reads finished span fields during payload encoding. +func getSpanKind(s *Span) string { return s.meta[ext.SpanKind] } + +// ----------------------------------------------------------------------------- +// Attribute conversion (DD → OTLP KeyValue / AnyValue) +// ----------------------------------------------------------------------------- + +// +checklocksignore — Post-finish: reads finished span fields during payload encoding. +func convertSpanAttributes(s *Span) []*otlpcommon.KeyValue { + attributes := convertMapToOTLPAttributesString(s.meta) + for key, value := range s.metrics { + attributes = append(attributes, otlpKeyValue(key, otlpDoubleValue(value))) + } + return attributes +} + +func convertMapToOTLPAttributesString(ddAttributes map[string]string) []*otlpcommon.KeyValue { + out := make([]*otlpcommon.KeyValue, 0, len(ddAttributes)) + for key, value := range ddAttributes { + out = append(out, otlpKeyValue(key, otlpStringValue(value))) + } + return out +} + +func convertEventAttributes(ddAttributes map[string]*spanEventAttribute) []*otlpcommon.KeyValue { + out := make([]*otlpcommon.KeyValue, 0, len(ddAttributes)) + for key, value := range ddAttributes { + switch value.Type { + case spanEventAttributeTypeString: + out = append(out, otlpKeyValue(key, otlpStringValue(value.StringValue))) + case spanEventAttributeTypeBool: + out = append(out, otlpKeyValue(key, otlpBoolValue(value.BoolValue))) + case spanEventAttributeTypeDouble: + out = append(out, otlpKeyValue(key, otlpDoubleValue(value.DoubleValue))) + case spanEventAttributeTypeInt: + out = append(out, otlpKeyValue(key, otlpIntValue(value.IntValue))) + case spanEventAttributeTypeArray: + out = append(out, otlpKeyValue(key, otlpArrayValue(value.ArrayValue))) + } + } + return out +} + +// --- AnyValue helpers --- + +func otlpKeyValue(key string, value *otlpcommon.AnyValue) *otlpcommon.KeyValue { + if value == nil { + return nil + } + return &otlpcommon.KeyValue{Key: key, Value: value} +} + +func otlpStringValue(s string) *otlpcommon.AnyValue { + return &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_StringValue{StringValue: s}} +} + +func otlpDoubleValue(d float64) *otlpcommon.AnyValue { + return &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_DoubleValue{DoubleValue: d}} +} + +func otlpBoolValue(b bool) *otlpcommon.AnyValue { + return &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_BoolValue{BoolValue: b}} +} + +func otlpIntValue(i int64) *otlpcommon.AnyValue { + return &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_IntValue{IntValue: i}} +} + +func otlpArrayValue(arr *spanEventArrayAttribute) *otlpcommon.AnyValue { + if arr == nil || len(arr.Values) == 0 { + return &otlpcommon.AnyValue{} + } + values := make([]*otlpcommon.AnyValue, 0, len(arr.Values)) + for _, v := range arr.Values { + if av := spanEventArrayAttributeValueToAnyValue(v); av != nil { + values = append(values, av) + } + } + return &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_ArrayValue{ArrayValue: &otlpcommon.ArrayValue{Values: values}}} +} + +func spanEventArrayAttributeValueToAnyValue(v *spanEventArrayAttributeValue) *otlpcommon.AnyValue { + if v == nil { + return nil + } + switch v.Type { + case spanEventArrayAttributeValueTypeString: + return otlpStringValue(v.StringValue) + case spanEventArrayAttributeValueTypeBool: + return otlpBoolValue(v.BoolValue) + case spanEventArrayAttributeValueTypeInt: + return otlpIntValue(v.IntValue) + case spanEventArrayAttributeValueTypeDouble: + return otlpDoubleValue(v.DoubleValue) + default: + return nil + } } diff --git a/ddtrace/tracer/transport.go b/ddtrace/tracer/transport.go index b7f7009b487..9415ac06e0b 100644 --- a/ddtrace/tracer/transport.go +++ b/ddtrace/tracer/transport.go @@ -46,7 +46,7 @@ const ( // transport is an interface for communicating data to the agent. type transport interface { - // send sends the payload p to the agent using the transport set up. + // send sends the msgpack-encoded payload p to the agent using the transport set up. // It returns a non-nil response body when no error occurred. send(p payload) (body io.ReadCloser, err error) // sendStats sends the given stats payload to the agent. @@ -56,6 +56,13 @@ type transport interface { endpoint() string } +// otlpSender is a narrow interface for sending protobuf-encoded OTLP trace payloads. +// It is satisfied by httpTransport and used by otlpTraceWriter to avoid coupling +// the OTLP export path to the full transport interface. +type otlpSender interface { + sendOTLP(data []byte) (body io.ReadCloser, err error) +} + type httpTransport struct { traceURL string // the delivery URL for traces statsURL string // the delivery URL for stats @@ -187,6 +194,29 @@ func (t *httpTransport) send(p payload) (body io.ReadCloser, err error) { return response.Body, nil } +// sendOTLP sends a protobuf-encoded OTLP payload to the configured trace endpoint. +// Content-Length is set automatically by net/http because the body is a *bytes.Reader. +func (t *httpTransport) sendOTLP(p []byte) (body io.ReadCloser, err error) { + req, err := http.NewRequest("POST", t.traceURL, bytes.NewReader(p)) + if err != nil { + return nil, fmt.Errorf("cannot create http request: %s", err) + } + for header, value := range t.headers { + req.Header.Set(header, value) + } + response, err := t.client.Do(req) + // TODO: t.statsd will point to an agent address that is probably not in use; we need to decide what to do with these metrics. + if err != nil { + // reportAPIErrorsMetric(response, err, tracesAPIPath) + return nil, err + } + if code := response.StatusCode; code >= 400 { + // reportAPIErrorsMetric(response, err, tracesAPIPath) + return nil, fmt.Errorf("status code: %d", code) + } + return response.Body, nil +} + func reportAPIErrorsMetric(response *http.Response, err error, endpoint string) { if t, ok := getGlobalTracer().(*tracer); ok { var reason string From a934fdd9cc960381f45b5b7360df5f1c3e4e51b5 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Tue, 24 Mar 2026 16:17:59 -0400 Subject: [PATCH 17/58] rename file payload_otlp_convert.go -> span_to_otlp.go; and introduce span_to_otlp_test.go --- ...ayload_otlp_convert.go => span_to_otlp.go} | 0 ddtrace/tracer/span_to_otlp_test.go | 230 ++++++++++++++++++ 2 files changed, 230 insertions(+) rename ddtrace/tracer/{payload_otlp_convert.go => span_to_otlp.go} (100%) create mode 100644 ddtrace/tracer/span_to_otlp_test.go diff --git a/ddtrace/tracer/payload_otlp_convert.go b/ddtrace/tracer/span_to_otlp.go similarity index 100% rename from ddtrace/tracer/payload_otlp_convert.go rename to ddtrace/tracer/span_to_otlp.go diff --git a/ddtrace/tracer/span_to_otlp_test.go b/ddtrace/tracer/span_to_otlp_test.go new file mode 100644 index 00000000000..5bc41528253 --- /dev/null +++ b/ddtrace/tracer/span_to_otlp_test.go @@ -0,0 +1,230 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016 Datadog, Inc. + +package tracer + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + otlpcommon "go.opentelemetry.io/proto/otlp/common/v1" + otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" + + "github.com/DataDog/dd-trace-go/v2/ddtrace/ext" +) + +func TestConvertSpan(t *testing.T) { + s := newSpan("op", "svc", "my-resource", 100, 200, 50) + s.start = 1000 + s.duration = 100 + s.meta[ext.SpanKind] = ext.SpanKindServer + s.meta["meta.key"] = "meta.val" + s.metrics["metric.key"] = 42.5 + s.error = 1 + s.meta[ext.ErrorMsg] = "something failed" + + otlp := convertSpan(s) + + assert.Equal(t, "my-resource", otlp.Name) + assert.Equal(t, uint64(1000), otlp.StartTimeUnixNano) + assert.Equal(t, uint64(1100), otlp.EndTimeUnixNano) + assert.Equal(t, otlptrace.Span_SPAN_KIND_SERVER, otlp.Kind) + require.NotNil(t, otlp.Status) + assert.Equal(t, otlptrace.Status_STATUS_CODE_ERROR, otlp.Status.Code) + assert.Equal(t, "something failed", otlp.Status.Message) + + attrs := keyValuesToMap(otlp.Attributes) + assert.Equal(t, "meta.val", attrs["meta.key"]) + assert.Equal(t, 42.5, attrs["metric.key"]) +} + +func TestConvertSpanKind(t *testing.T) { + tests := []struct { + dd string + want otlptrace.Span_SpanKind + }{ + {ext.SpanKindInternal, otlptrace.Span_SPAN_KIND_INTERNAL}, + {ext.SpanKindServer, otlptrace.Span_SPAN_KIND_SERVER}, + {ext.SpanKindClient, otlptrace.Span_SPAN_KIND_CLIENT}, + {ext.SpanKindProducer, otlptrace.Span_SPAN_KIND_PRODUCER}, + {ext.SpanKindConsumer, otlptrace.Span_SPAN_KIND_CONSUMER}, + {"", otlptrace.Span_SPAN_KIND_INTERNAL}, + {"unknown", otlptrace.Span_SPAN_KIND_INTERNAL}, + } + for _, tt := range tests { + t.Run(tt.dd, func(t *testing.T) { + got := convertSpanKind(tt.dd) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestConvertSpanStatus(t *testing.T) { + t.Run("unset", func(t *testing.T) { + s := newBasicSpan("op") + s.error = 0 + st := convertSpanStatus(s) + require.NotNil(t, st) + assert.Equal(t, otlptrace.Status_STATUS_CODE_UNSET, st.Code) + }) + + t.Run("error", func(t *testing.T) { + s := newBasicSpan("op") + s.error = 1 + s.meta = map[string]string{ext.ErrorMsg: "err msg"} + st := convertSpanStatus(s) + require.NotNil(t, st) + assert.Equal(t, otlptrace.Status_STATUS_CODE_ERROR, st.Code) + assert.Equal(t, "err msg", st.Message) + }) +} + +func TestConvertSpanAttributes(t *testing.T) { + s := newBasicSpan("op") + s.meta = map[string]string{"tag": "val", "env": "test"} + s.metrics = map[string]float64{"count": 10, "rate": 0.5} + + attrs := convertSpanAttributes(s) + m := keyValuesToMap(attrs) + assert.Equal(t, "val", m["tag"]) + assert.Equal(t, "test", m["env"]) + assert.Equal(t, 10.0, m["count"]) + assert.Equal(t, 0.5, m["rate"]) +} + +func TestConvertMapToOTLPAttributesString(t *testing.T) { + dd := map[string]string{"k1": "v1", "k2": "v2"} + otlp := convertMapToOTLPAttributesString(dd) + require.Len(t, otlp, 2) + m := keyValuesToMap(otlp) + assert.Equal(t, "v1", m["k1"]) + assert.Equal(t, "v2", m["k2"]) +} + +func TestConvertMapToOTLPAttributesString_EmptyNil(t *testing.T) { + assert.Empty(t, convertMapToOTLPAttributesString(nil)) + assert.Empty(t, convertMapToOTLPAttributesString(map[string]string{})) +} + +func TestConvertEvents(t *testing.T) { + s := newBasicSpan("op") + s.spanEvents = []spanEvent{ + { + Name: "event1", + TimeUnixNano: 1000, + Attributes: map[string]*spanEventAttribute{ + "attr1": {Type: spanEventAttributeTypeString, StringValue: "s1"}, + }, + }, + { + Name: "event2", + TimeUnixNano: 2000, + Attributes: nil, + }, + } + + events := convertEvents(s) + require.Len(t, events, 2) + assert.Equal(t, "event1", events[0].Name) + assert.Equal(t, uint64(1000), events[0].TimeUnixNano) + require.Len(t, events[0].Attributes, 1) + assert.Equal(t, "attr1", events[0].Attributes[0].Key) + assert.Equal(t, "s1", events[0].Attributes[0].Value.GetStringValue()) + + assert.Equal(t, "event2", events[1].Name) + assert.Equal(t, uint64(2000), events[1].TimeUnixNano) + assert.Empty(t, events[1].Attributes) +} + +func TestConvertEventAttributes(t *testing.T) { + dd := map[string]*spanEventAttribute{ + "str": {Type: spanEventAttributeTypeString, StringValue: "x"}, + "bool": {Type: spanEventAttributeTypeBool, BoolValue: true}, + "int": {Type: spanEventAttributeTypeInt, IntValue: -7}, + "float": {Type: spanEventAttributeTypeDouble, DoubleValue: 3.14}, + "arr": { + Type: spanEventAttributeTypeArray, + ArrayValue: &spanEventArrayAttribute{ + Values: []*spanEventArrayAttributeValue{ + {Type: spanEventArrayAttributeValueTypeString, StringValue: "elem1"}, + {Type: spanEventArrayAttributeValueTypeInt, IntValue: 99}, + }, + }, + }, + } + otlp := convertEventAttributes(dd) + require.Len(t, otlp, 5) + + m := keyValuesToMap(otlp) + assert.Equal(t, "x", m["str"]) + assert.Equal(t, true, m["bool"]) + assert.Equal(t, int64(-7), m["int"]) + assert.Equal(t, 3.14, m["float"]) + + // Array: assert outside keyValuesToMap (it doesn't flatten arrays) + var arrKV *otlpcommon.KeyValue + for _, kv := range otlp { + if kv != nil && kv.Key == "arr" { + arrKV = kv + break + } + } + require.NotNil(t, arrKV) + av := arrKV.Value.GetArrayValue() + require.NotNil(t, av) + require.Len(t, av.Values, 2) + assert.Equal(t, "elem1", av.Values[0].GetStringValue()) + assert.Equal(t, int64(99), av.Values[1].GetIntValue()) +} + +func TestConvertEventAttributes_NilEmpty(t *testing.T) { + assert.Empty(t, convertEventAttributes(nil)) + assert.Empty(t, convertEventAttributes(map[string]*spanEventAttribute{})) +} + +func TestConvertSpanLinks(t *testing.T) { + links := []SpanLink{ + {TraceID: 1, SpanID: 10, Attributes: map[string]string{"k": "v"}, Tracestate: "ts", Flags: 1}, + {TraceID: 2, SpanID: 20}, + } + otlp := convertSpanLinks(links) + require.Len(t, otlp, 2) + attrs := keyValuesToMap(otlp[0].Attributes) + assert.Equal(t, "v", attrs["k"]) + assert.Equal(t, "ts", otlp[0].TraceState) + assert.Equal(t, uint32(1), otlp[0].Flags) + assert.Empty(t, otlp[1].Attributes) +} + +func TestConvertSpanLinks_EmptyNil(t *testing.T) { + assert.Empty(t, convertSpanLinks(nil)) + assert.Empty(t, convertSpanLinks([]SpanLink{})) +} + +// keyValuesToMap converts []*otlpcommon.KeyValue into a map for easier assertion. +// Values are returned as any (string or float64 for double). +func keyValuesToMap(kvs []*otlpcommon.KeyValue) map[string]any { + m := make(map[string]any) + for _, kv := range kvs { + if kv == nil || kv.Value == nil { + continue + } + switch v := kv.Value.Value.(type) { + case *otlpcommon.AnyValue_StringValue: + m[kv.Key] = v.StringValue + case *otlpcommon.AnyValue_DoubleValue: + m[kv.Key] = v.DoubleValue + case *otlpcommon.AnyValue_IntValue: + m[kv.Key] = v.IntValue + case *otlpcommon.AnyValue_BoolValue: + m[kv.Key] = v.BoolValue + default: + m[kv.Key] = nil + } + } + return m +} From 2cfa217ab7bbfa878079c5cf16daa36e994484d5 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Tue, 24 Mar 2026 16:26:10 -0400 Subject: [PATCH 18/58] introduce otlp_writer_test.go: basic otlp writer tests --- ddtrace/tracer/otlp_writer_test.go | 279 +++++++++++++++++++++++++++++ 1 file changed, 279 insertions(+) create mode 100644 ddtrace/tracer/otlp_writer_test.go diff --git a/ddtrace/tracer/otlp_writer_test.go b/ddtrace/tracer/otlp_writer_test.go new file mode 100644 index 00000000000..16be3feb473 --- /dev/null +++ b/ddtrace/tracer/otlp_writer_test.go @@ -0,0 +1,279 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016 Datadog, Inc. + +package tracer + +import ( + "errors" + "fmt" + "io" + "sync" + "sync/atomic" + "testing" + "time" + + internalconfig "github.com/DataDog/dd-trace-go/v2/internal/config" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" + "google.golang.org/protobuf/proto" +) + +// mockOTLPSender records sendOTLP calls for test assertions. +type mockOTLPSender struct { + mu sync.Mutex + calls [][]byte + failCount int + sendAttempts int +} + +func (m *mockOTLPSender) sendOTLP(data []byte) (io.ReadCloser, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.sendAttempts++ + if m.failCount > 0 { + m.failCount-- + return nil, errors.New("send failed") + } + cp := make([]byte, len(data)) + copy(cp, data) + m.calls = append(m.calls, cp) + return nil, nil +} + +func (m *mockOTLPSender) getSendAttempts() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.sendAttempts +} + +func (m *mockOTLPSender) getCalls() [][]byte { + m.mu.Lock() + defer m.mu.Unlock() + return m.calls +} + +func newTestOTLPWriter(t *testing.T, sender *mockOTLPSender, opts ...StartOption) *otlpTraceWriter { + t.Helper() + cfg, err := newTestConfig(append(opts, func(c *config) { + c.transport = &simpleTransport{} + })...) + require.NoError(t, err) + return &otlpTraceWriter{ + config: cfg, + sender: sender, + resource: buildResource(cfg.internalConfig), + scope: nil, + spans: make([]*otlptrace.Span, 0), + climit: make(chan struct{}, concurrentConnectionLimit), + } +} + +func TestOTLPWriterImplementsTraceWriter(t *testing.T) { + assert.Implements(t, (*traceWriter)(nil), &otlpTraceWriter{}) +} + +func TestOTLPWriterAdd(t *testing.T) { + sender := &mockOTLPSender{} + w := newTestOTLPWriter(t, sender) + + spans := []*Span{ + newSpan("op1", "svc", "res1", 1, 1, 0), + newSpan("op2", "svc", "res2", 2, 1, 1), + } + w.add(spans) + + w.mu.Lock() + assert.Equal(t, 2, len(w.spans)) + w.mu.Unlock() +} + +func TestOTLPWriterAddMultiple(t *testing.T) { + sender := &mockOTLPSender{} + w := newTestOTLPWriter(t, sender) + + w.add([]*Span{newSpan("op1", "svc", "res", 1, 1, 0)}) + w.add([]*Span{newSpan("op2", "svc", "res", 2, 1, 0)}) + w.add([]*Span{newSpan("op3", "svc", "res", 3, 1, 0)}) + + w.mu.Lock() + assert.Equal(t, 3, len(w.spans)) + w.mu.Unlock() +} + +func TestOTLPWriterFlushEmpty(t *testing.T) { + sender := &mockOTLPSender{} + w := newTestOTLPWriter(t, sender) + + w.flush() + w.wg.Wait() + + assert.Equal(t, 0, sender.getSendAttempts()) +} + +func TestOTLPWriterFlush(t *testing.T) { + sender := &mockOTLPSender{} + w := newTestOTLPWriter(t, sender) + + w.add([]*Span{ + newSpan("op1", "svc", "res", 1, 1, 0), + newSpan("op2", "svc", "res", 2, 1, 0), + }) + w.flush() + w.wg.Wait() + + calls := sender.getCalls() + require.Equal(t, 1, len(calls)) + + var tracesData otlptrace.TracesData + err := proto.Unmarshal(calls[0], &tracesData) + require.NoError(t, err) + + rs := tracesData.ResourceSpans + require.Equal(t, 1, len(rs)) + require.Equal(t, 1, len(rs[0].ScopeSpans)) + assert.Equal(t, 2, len(rs[0].ScopeSpans[0].Spans)) +} + +func TestOTLPWriterFlushClearsSpans(t *testing.T) { + sender := &mockOTLPSender{} + w := newTestOTLPWriter(t, sender) + + w.add([]*Span{newSpan("op1", "svc", "res", 1, 1, 0)}) + w.flush() + w.wg.Wait() + + w.mu.Lock() + assert.Equal(t, 0, len(w.spans)) + w.mu.Unlock() + + // Second flush should be a no-op + w.flush() + w.wg.Wait() + assert.Equal(t, 1, sender.getSendAttempts()) +} + +func TestOTLPWriterFlushRetries(t *testing.T) { + testcases := []struct { + configRetries int + failCount int + tracesSent bool + expAttempts int + }{ + {configRetries: 0, failCount: 0, tracesSent: true, expAttempts: 1}, + {configRetries: 0, failCount: 1, tracesSent: false, expAttempts: 1}, + + {configRetries: 1, failCount: 0, tracesSent: true, expAttempts: 1}, + {configRetries: 1, failCount: 1, tracesSent: true, expAttempts: 2}, + {configRetries: 1, failCount: 2, tracesSent: false, expAttempts: 2}, + + {configRetries: 2, failCount: 0, tracesSent: true, expAttempts: 1}, + {configRetries: 2, failCount: 1, tracesSent: true, expAttempts: 2}, + {configRetries: 2, failCount: 2, tracesSent: true, expAttempts: 3}, + {configRetries: 2, failCount: 3, tracesSent: false, expAttempts: 3}, + } + + for _, tc := range testcases { + name := fmt.Sprintf("retries=%d/fails=%d", tc.configRetries, tc.failCount) + t.Run(name, func(t *testing.T) { + sender := &mockOTLPSender{failCount: tc.failCount} + w := newTestOTLPWriter(t, sender, func(c *config) { + c.sendRetries = tc.configRetries + c.internalConfig.SetRetryInterval(time.Millisecond, internalconfig.OriginCode) + }) + + w.add([]*Span{newSpan("op", "svc", "res", 1, 1, 0)}) + w.flush() + w.wg.Wait() + + assert.Equal(t, tc.expAttempts, sender.getSendAttempts()) + assert.Equal(t, tc.tracesSent, len(sender.getCalls()) > 0) + }) + } +} + +func TestOTLPWriterStop(t *testing.T) { + sender := &mockOTLPSender{} + w := newTestOTLPWriter(t, sender) + + w.add([]*Span{newSpan("op", "svc", "res", 1, 1, 0)}) + w.stop() + + assert.Equal(t, 1, len(sender.getCalls())) +} + +func TestOTLPWriterConcurrency(t *testing.T) { + sender := &mockOTLPSender{} + w := newTestOTLPWriter(t, sender) + + const numAdders = 20 + const spansPerAdder = 50 + const numFlushers = 10 + + start := make(chan struct{}) + var wg sync.WaitGroup + + var spansAdded int32 + + for range numAdders { + wg.Add(1) + go func() { + defer wg.Done() + <-start + for range spansPerAdder { + w.add([]*Span{newSpan("op", "svc", "res", randUint64(), randUint64(), 0)}) + atomic.AddInt32(&spansAdded, 1) + } + }() + } + + for range numFlushers { + wg.Add(1) + go func() { + defer wg.Done() + <-start + for range 10 { + w.flush() + } + }() + } + + close(start) + wg.Wait() + + w.stop() + + assert.Equal(t, int32(numAdders*spansPerAdder), atomic.LoadInt32(&spansAdded)) + + // Verify all sent payloads are valid protobuf + totalSpans := 0 + for _, data := range sender.getCalls() { + var td otlptrace.TracesData + err := proto.Unmarshal(data, &td) + require.NoError(t, err) + for _, rs := range td.ResourceSpans { + for _, ss := range rs.ScopeSpans { + totalSpans += len(ss.Spans) + } + } + } + assert.Equal(t, numAdders*spansPerAdder, totalSpans) +} + +func TestOTLPWriterNoopSender(t *testing.T) { + cfg, err := newTestConfig(func(c *config) { + c.transport = &simpleTransport{} + }) + require.NoError(t, err) + + w := newOTLPTraceWriter(cfg) + + // simpleTransport doesn't implement otlpSender, so noopOTLPSender should be used + w.add([]*Span{newSpan("op", "svc", "res", 1, 1, 0)}) + w.flush() + w.wg.Wait() + // No panic, no actual send — the noop sender returns an error that gets logged +} From 7bba29d6780ac0eae3b547f44f4923457e456007 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Tue, 24 Mar 2026 16:36:21 -0400 Subject: [PATCH 19/58] Fix: only export sampled spans, and correct telemetry.sdk.version value --- ddtrace/tracer/otlp_writer.go | 4 +++- ddtrace/tracer/span_to_otlp.go | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ddtrace/tracer/otlp_writer.go b/ddtrace/tracer/otlp_writer.go index 3227a6b830e..1cecd02787c 100644 --- a/ddtrace/tracer/otlp_writer.go +++ b/ddtrace/tracer/otlp_writer.go @@ -63,7 +63,9 @@ func (w *otlpTraceWriter) add(spanList []*Span) { defer w.mu.Unlock() w.spans = slices.Grow(w.spans, len(spanList)) for _, span := range spanList { - w.spans = append(w.spans, convertSpan(span)) + if otlpSpan := convertSpan(span); otlpSpan != nil { + w.spans = append(w.spans, otlpSpan) + } } } diff --git a/ddtrace/tracer/span_to_otlp.go b/ddtrace/tracer/span_to_otlp.go index 631c9f5389f..0fd00217ae2 100644 --- a/ddtrace/tracer/span_to_otlp.go +++ b/ddtrace/tracer/span_to_otlp.go @@ -9,6 +9,7 @@ import ( "encoding/binary" internalconfig "github.com/DataDog/dd-trace-go/v2/internal/config" + "github.com/DataDog/dd-trace-go/v2/internal/version" otlpcommon "go.opentelemetry.io/proto/otlp/common/v1" otlpresource "go.opentelemetry.io/proto/otlp/resource/v1" otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" @@ -30,10 +31,10 @@ func buildResource(cfg *internalconfig.Config) *otlpresource.Resource { otlpKeyValue("service.name", otlpStringValue(cfg.ServiceName())), otlpKeyValue("telemetry.sdk.language", otlpStringValue("go")), otlpKeyValue("telemetry.sdk.name", otlpStringValue("datadog")), - otlpKeyValue("telemetry.sdk.version", otlpStringValue(cfg.Version())), + otlpKeyValue("telemetry.sdk.version", otlpStringValue(version.Tag)), } if v := cfg.Env(); v != "" { - attrs = append(attrs, otlpKeyValue("deployment.environment", otlpStringValue(v))) + attrs = append(attrs, otlpKeyValue("deployment.environment.name", otlpStringValue(v))) } if v := cfg.Version(); v != "" { attrs = append(attrs, otlpKeyValue("service.version", otlpStringValue(v))) @@ -47,9 +48,13 @@ func buildResource(cfg *internalconfig.Config) *otlpresource.Resource { // +checklocksignore — Post-finish: reads finished span fields during payload encoding. func convertSpan(s *Span) *otlptrace.Span { + if p, ok := s.context.SamplingPriority(); ok && p < ext.PriorityAutoKeep { + return nil + } return &otlptrace.Span{ TraceId: convertTraceID(s.context.traceID.Upper(), s.context.traceID.Lower()), SpanId: convertSpanID(s.spanID), + ParentSpanId: convertSpanID(s.parentID), Name: s.resource, Kind: convertSpanKind(getSpanKind(s)), StartTimeUnixNano: uint64(s.start), From 89cb7e6743b979dbdc270853d8f6a427d1a95c9c Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Tue, 24 Mar 2026 16:36:42 -0400 Subject: [PATCH 20/58] Test buildResource --- ddtrace/tracer/span_to_otlp_test.go | 87 +++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/ddtrace/tracer/span_to_otlp_test.go b/ddtrace/tracer/span_to_otlp_test.go index 5bc41528253..badfefc5c47 100644 --- a/ddtrace/tracer/span_to_otlp_test.go +++ b/ddtrace/tracer/span_to_otlp_test.go @@ -6,6 +6,7 @@ package tracer import ( + "encoding/binary" "testing" "github.com/stretchr/testify/assert" @@ -14,9 +15,50 @@ import ( otlpcommon "go.opentelemetry.io/proto/otlp/common/v1" otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" + internalconfig "github.com/DataDog/dd-trace-go/v2/internal/config" + "github.com/DataDog/dd-trace-go/v2/internal/version" + "github.com/DataDog/dd-trace-go/v2/ddtrace/ext" + "github.com/DataDog/dd-trace-go/v2/internal/samplernames" ) +func TestBuildResource(t *testing.T) { + t.Run("nil config", func(t *testing.T) { + r := buildResource(nil) + require.NotNil(t, r) + assert.Empty(t, r.Attributes) + }) + + t.Run("populated", func(t *testing.T) { + cfg := internalconfig.CreateNew() + cfg.SetServiceName("my-service", internalconfig.OriginCode) + cfg.SetEnv("prod", internalconfig.OriginCode) + cfg.SetVersion("1.2.3", internalconfig.OriginCode) + + r := buildResource(cfg) + attrs := keyValuesToMap(r.Attributes) + + assert.Equal(t, "my-service", attrs["service.name"]) + assert.Equal(t, "prod", attrs["deployment.environment.name"]) + assert.Equal(t, "1.2.3", attrs["service.version"]) + assert.Equal(t, "go", attrs["telemetry.sdk.language"]) + assert.Equal(t, "datadog", attrs["telemetry.sdk.name"]) + assert.Equal(t, version.Tag, attrs["telemetry.sdk.version"]) + }) + + t.Run("optional fields omitted when empty", func(t *testing.T) { + cfg := internalconfig.CreateNew() + cfg.SetServiceName("svc", internalconfig.OriginCode) + + r := buildResource(cfg) + attrs := keyValuesToMap(r.Attributes) + + assert.Equal(t, "svc", attrs["service.name"]) + _, hasEnv := attrs["deployment.environment.name"] + assert.False(t, hasEnv, "deployment.environment.name should be absent when env is empty") + }) +} + func TestConvertSpan(t *testing.T) { s := newSpan("op", "svc", "my-resource", 100, 200, 50) s.start = 1000 @@ -28,20 +70,65 @@ func TestConvertSpan(t *testing.T) { s.meta[ext.ErrorMsg] = "something failed" otlp := convertSpan(s) + require.NotNil(t, otlp) + // DD resource → OTLP name (spec: "resource field must be encoded as the OTLP span's name field") assert.Equal(t, "my-resource", otlp.Name) + assert.Equal(t, uint64(1000), otlp.StartTimeUnixNano) assert.Equal(t, uint64(1100), otlp.EndTimeUnixNano) assert.Equal(t, otlptrace.Span_SPAN_KIND_SERVER, otlp.Kind) + + // parent_span_id + assert.Equal(t, uint64(50), binary.BigEndian.Uint64(otlp.ParentSpanId)) + + // trace_id and span_id are populated + assert.Len(t, otlp.TraceId, 16) + assert.Len(t, otlp.SpanId, 8) + assert.Equal(t, uint64(100), binary.BigEndian.Uint64(otlp.SpanId)) + + // Status require.NotNil(t, otlp.Status) assert.Equal(t, otlptrace.Status_STATUS_CODE_ERROR, otlp.Status.Code) assert.Equal(t, "something failed", otlp.Status.Message) + // Attributes: meta as strings, metrics as doubles attrs := keyValuesToMap(otlp.Attributes) assert.Equal(t, "meta.val", attrs["meta.key"]) assert.Equal(t, 42.5, attrs["metric.key"]) } +func TestConvertSpanFiltersUnsampled(t *testing.T) { + tests := []struct { + name string + priority int + wantNil bool + }{ + {"auto-reject", ext.PriorityAutoReject, true}, + {"auto-keep", ext.PriorityAutoKeep, false}, + {"user-keep", ext.PriorityUserKeep, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newSpan("op", "svc", "res", 1, 1, 0) + s.context.setSamplingPriority(tt.priority, samplernames.Unknown) + result := convertSpan(s) + if tt.wantNil { + assert.Nil(t, result) + } else { + assert.NotNil(t, result) + } + }) + } +} + +func TestConvertSpanPriorityUnset(t *testing.T) { + s := newSpan("op", "svc", "res", 1, 1, 0) + // priority is not set — span should be included (not dropped) + result := convertSpan(s) + assert.NotNil(t, result) +} + func TestConvertSpanKind(t *testing.T) { tests := []struct { dd string From da1efd4b43bbb9a8facc015a2a1a9eb866baa194 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Tue, 24 Mar 2026 16:46:09 -0400 Subject: [PATCH 21/58] implement edge case: add 'service.name' attribute to span when span's service name differs from global service name --- ddtrace/tracer/otlp_writer.go | 3 +- ddtrace/tracer/span_to_otlp.go | 9 ++++-- ddtrace/tracer/span_to_otlp_test.go | 46 ++++++++++++++++++++++++++--- 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/ddtrace/tracer/otlp_writer.go b/ddtrace/tracer/otlp_writer.go index 1cecd02787c..a40e81f0fc4 100644 --- a/ddtrace/tracer/otlp_writer.go +++ b/ddtrace/tracer/otlp_writer.go @@ -59,11 +59,12 @@ func newOTLPTraceWriter(c *config) *otlpTraceWriter { } func (w *otlpTraceWriter) add(spanList []*Span) { + defaultServiceName := w.config.internalConfig.ServiceName() w.mu.Lock() defer w.mu.Unlock() w.spans = slices.Grow(w.spans, len(spanList)) for _, span := range spanList { - if otlpSpan := convertSpan(span); otlpSpan != nil { + if otlpSpan := convertSpan(span, defaultServiceName); otlpSpan != nil { w.spans = append(w.spans, otlpSpan) } } diff --git a/ddtrace/tracer/span_to_otlp.go b/ddtrace/tracer/span_to_otlp.go index 0fd00217ae2..62aab20bbc9 100644 --- a/ddtrace/tracer/span_to_otlp.go +++ b/ddtrace/tracer/span_to_otlp.go @@ -47,7 +47,7 @@ func buildResource(cfg *internalconfig.Config) *otlpresource.Resource { // ----------------------------------------------------------------------------- // +checklocksignore — Post-finish: reads finished span fields during payload encoding. -func convertSpan(s *Span) *otlptrace.Span { +func convertSpan(s *Span, defaultServiceName string) *otlptrace.Span { if p, ok := s.context.SamplingPriority(); ok && p < ext.PriorityAutoKeep { return nil } @@ -59,7 +59,7 @@ func convertSpan(s *Span) *otlptrace.Span { Kind: convertSpanKind(getSpanKind(s)), StartTimeUnixNano: uint64(s.start), EndTimeUnixNano: uint64(s.start + s.duration), - Attributes: convertSpanAttributes(s), + Attributes: convertSpanAttributes(s, defaultServiceName), Events: convertEvents(s), Links: convertSpanLinks(s.spanLinks), Status: convertSpanStatus(s), @@ -149,11 +149,14 @@ func getSpanKind(s *Span) string { return s.meta[ext.SpanKind] } // ----------------------------------------------------------------------------- // +checklocksignore — Post-finish: reads finished span fields during payload encoding. -func convertSpanAttributes(s *Span) []*otlpcommon.KeyValue { +func convertSpanAttributes(s *Span, defaultServiceName string) []*otlpcommon.KeyValue { attributes := convertMapToOTLPAttributesString(s.meta) for key, value := range s.metrics { attributes = append(attributes, otlpKeyValue(key, otlpDoubleValue(value))) } + if s.service != defaultServiceName { + attributes = append(attributes, otlpKeyValue("service.name", otlpStringValue(s.service))) + } return attributes } diff --git a/ddtrace/tracer/span_to_otlp_test.go b/ddtrace/tracer/span_to_otlp_test.go index badfefc5c47..6bc6ef3b0ce 100644 --- a/ddtrace/tracer/span_to_otlp_test.go +++ b/ddtrace/tracer/span_to_otlp_test.go @@ -56,6 +56,8 @@ func TestBuildResource(t *testing.T) { assert.Equal(t, "svc", attrs["service.name"]) _, hasEnv := attrs["deployment.environment.name"] assert.False(t, hasEnv, "deployment.environment.name should be absent when env is empty") + _, hasVer := attrs["service.version"] + assert.False(t, hasVer, "service.version should be absent when version is empty") }) } @@ -69,7 +71,7 @@ func TestConvertSpan(t *testing.T) { s.error = 1 s.meta[ext.ErrorMsg] = "something failed" - otlp := convertSpan(s) + otlp := convertSpan(s, "svc") require.NotNil(t, otlp) // DD resource → OTLP name (spec: "resource field must be encoded as the OTLP span's name field") @@ -112,7 +114,7 @@ func TestConvertSpanFiltersUnsampled(t *testing.T) { t.Run(tt.name, func(t *testing.T) { s := newSpan("op", "svc", "res", 1, 1, 0) s.context.setSamplingPriority(tt.priority, samplernames.Unknown) - result := convertSpan(s) + result := convertSpan(s, "svc") if tt.wantNil { assert.Nil(t, result) } else { @@ -125,10 +127,29 @@ func TestConvertSpanFiltersUnsampled(t *testing.T) { func TestConvertSpanPriorityUnset(t *testing.T) { s := newSpan("op", "svc", "res", 1, 1, 0) // priority is not set — span should be included (not dropped) - result := convertSpan(s) + result := convertSpan(s, "") assert.NotNil(t, result) } +func TestConvertSpanServiceNameOverride(t *testing.T) { + t.Run("same as default - no service.name attribute", func(t *testing.T) { + s := newSpan("op", "my-service", "res", 1, 1, 0) + otlp := convertSpan(s, "my-service") + require.NotNil(t, otlp) + attrs := keyValuesToMap(otlp.Attributes) + _, hasServiceName := attrs["service.name"] + assert.False(t, hasServiceName) + }) + + t.Run("different from default - service.name attribute added", func(t *testing.T) { + s := newSpan("op", "other-service", "res", 1, 1, 0) + otlp := convertSpan(s, "my-service") + require.NotNil(t, otlp) + attrs := keyValuesToMap(otlp.Attributes) + assert.Equal(t, "other-service", attrs["service.name"]) + }) +} + func TestConvertSpanKind(t *testing.T) { tests := []struct { dd string @@ -175,7 +196,7 @@ func TestConvertSpanAttributes(t *testing.T) { s.meta = map[string]string{"tag": "val", "env": "test"} s.metrics = map[string]float64{"count": 10, "rate": 0.5} - attrs := convertSpanAttributes(s) + attrs := convertSpanAttributes(s, "") m := keyValuesToMap(attrs) assert.Equal(t, "val", m["tag"]) assert.Equal(t, "test", m["env"]) @@ -183,6 +204,23 @@ func TestConvertSpanAttributes(t *testing.T) { assert.Equal(t, 0.5, m["rate"]) } +func TestConvertSpanAttributesServiceNameOverride(t *testing.T) { + t.Run("same as default - no attribute", func(t *testing.T) { + s := newSpan("op", "my-service", "res", 1, 1, 0) + attrs := convertSpanAttributes(s, "my-service") + m := keyValuesToMap(attrs) + _, hasServiceName := m["service.name"] + assert.False(t, hasServiceName, "service.name attribute should be absent when it matches the default") + }) + + t.Run("different from default - attribute added", func(t *testing.T) { + s := newSpan("op", "other-service", "res", 1, 1, 0) + attrs := convertSpanAttributes(s, "my-service") + m := keyValuesToMap(attrs) + assert.Equal(t, "other-service", m["service.name"]) + }) +} + func TestConvertMapToOTLPAttributesString(t *testing.T) { dd := map[string]string{"k1": "v1", "k2": "v2"} otlp := convertMapToOTLPAttributesString(dd) From 49053c19580b12b57b61053029c9cc4f1467eee9 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Wed, 25 Mar 2026 11:12:42 -0400 Subject: [PATCH 22/58] add comment about s.metaStruct --- ddtrace/tracer/span_to_otlp.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ddtrace/tracer/span_to_otlp.go b/ddtrace/tracer/span_to_otlp.go index 62aab20bbc9..3c2eadcdb00 100644 --- a/ddtrace/tracer/span_to_otlp.go +++ b/ddtrace/tracer/span_to_otlp.go @@ -157,6 +157,7 @@ func convertSpanAttributes(s *Span, defaultServiceName string) []*otlpcommon.Key if s.service != defaultServiceName { attributes = append(attributes, otlpKeyValue("service.name", otlpStringValue(s.service))) } + // TODO: add meta_struct attributes return attributes } From 378639fe78b2e625e9965840b4b4f017e4ed2151 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Wed, 25 Mar 2026 05:54:37 -0400 Subject: [PATCH 23/58] fix(ddtrace/tracer): pre-compute `_dd.p.dm` numeric value at write-time (#4576) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What does this PR do? Moves the `strconv.ParseInt` call for `_dd.p.dm` (decision maker) out of the v1 payload flush path and into the write path. A `dm uint32` field is added to the `trace` struct. It is kept in sync with `propagatingTags[keyDecisionMaker]` at all mutation sites (`setPropagatingTagLocked`, `unsetPropagatingTagLocked`, `replacePropagatingTags`). A shared `unsetPropagatingTagLocked` helper is introduced so the delete-and-clear logic is not duplicated between `unsetPropagatingTag` and `setSamplingPriorityLockedWithForce`. `payloadV1.push` now reads the pre-computed value via `trace.decisionMaker()` instead of fetching the string from the propagating tags map and parsing it on every flush. ### Motivation `payloadV1.push` is called once per finished trace chunk. On every call it was acquiring a read lock, doing a map lookup, and then running `strconv.ParseInt` — all to read a value that was set at most once or twice during the lifetime of the trace. Moving the parse to write-time eliminates the per-flush map lookup and parse, replacing them with a single scalar read behind an `RLock`. ### Reviewer's Checklist - [x] Changed code has unit tests for its functionality at or near 100% coverage. - [x] New code is free of linting errors. You can check this by running `make lint` locally. - [x] New code doesn't break existing tests. You can check this by running `make test` locally. - [x] Add an appropriate team label so this PR gets put in the right place for the release notes. - [x] All generated files are up to date. You can check this by running `make generate` locally. Co-authored-by: dario.castane --- ddtrace/tracer/payload_v1.go | 12 +-------- ddtrace/tracer/propagating_tags.go | 39 ++++++++++++++++++++++++++++++ ddtrace/tracer/spancontext.go | 13 +++++++--- ddtrace/tracer/spancontext_test.go | 30 +++++++++++++++++++++++ 4 files changed, 80 insertions(+), 14 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 520dac1a10c..ca70c6fc4c6 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -10,7 +10,6 @@ import ( "encoding/binary" "errors" "fmt" - "strconv" "sync" "sync/atomic" @@ -189,16 +188,7 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { // TODO(darccio): are we sure that origin will be shared across all the spans in the chunk? origin = span.Context().origin // +checklocksignore - Read-only after init. - if dm := span.context.trace.propagatingTag(keyDecisionMaker); dm != "" { - if v, err := strconv.ParseInt(dm, 10, 32); err == nil { - if v < 0 { - v = -v - } - sm = uint32(v) - } else { - log.Error("failed to convert decision maker to uint32: %s", err.Error()) - } - } + sm = span.context.trace.decisionMaker() } tc := traceChunk{ spans: t, diff --git a/ddtrace/tracer/propagating_tags.go b/ddtrace/tracer/propagating_tags.go index 5b595472912..ab1429b7309 100644 --- a/ddtrace/tracer/propagating_tags.go +++ b/ddtrace/tracer/propagating_tags.go @@ -6,6 +6,8 @@ package tracer import ( + "strconv" + "github.com/DataDog/dd-trace-go/v2/internal" "github.com/DataDog/dd-trace-go/v2/internal/locking/assert" "github.com/DataDog/dd-trace-go/v2/internal/log" @@ -61,13 +63,25 @@ func (t *trace) setPropagatingTagLocked(key, value string) { t.propagatingTags = make(map[string]string, 1) } t.propagatingTags[key] = value + if key == keyDecisionMaker { + t.dm = parseDecisionMaker(value) + } } // unsetPropagatingTag deletes the key/value pair from the trace's propagated tags. func (t *trace) unsetPropagatingTag(key string) { t.mu.Lock() defer t.mu.Unlock() + t.unsetPropagatingTagLocked(key) +} + +// +checklocks:t.mu +func (t *trace) unsetPropagatingTagLocked(key string) { + assert.RWMutexLocked(&t.mu) delete(t.propagatingTags, key) + if key == keyDecisionMaker { + t.dm = 0 + } } // iteratePropagatingTags allows safe iteration through the propagating tags of a trace. @@ -88,6 +102,11 @@ func (t *trace) replacePropagatingTags(tags map[string]string) { t.mu.Lock() defer t.mu.Unlock() t.propagatingTags = tags + if dm, ok := tags[keyDecisionMaker]; ok { + t.dm = parseDecisionMaker(dm) + } else { + t.dm = 0 + } } func (t *trace) propagatingTagsLen() int { @@ -95,3 +114,23 @@ func (t *trace) propagatingTagsLen() int { defer t.mu.RUnlock() return len(t.propagatingTags) } + +// parseDecisionMaker parses the decision maker string (e.g. "-4") into +// its absolute uint32 form for v1 protocol encoding. +func parseDecisionMaker(dm string) uint32 { + v, err := strconv.ParseInt(dm, 10, 32) + if err != nil { + log.Error("failed to convert decision maker to uint32: %s", err.Error()) + return 0 + } + if v < 0 { + v = -v + } + return uint32(v) +} + +func (t *trace) decisionMaker() uint32 { + t.mu.RLock() + defer t.mu.RUnlock() + return t.dm +} diff --git a/ddtrace/tracer/spancontext.go b/ddtrace/tracer/spancontext.go index 71336081960..4e0cdd184e0 100644 --- a/ddtrace/tracer/spancontext.go +++ b/ddtrace/tracer/spancontext.go @@ -196,8 +196,11 @@ func FromGenericCtx(c ddtrace.SpanContext) *SpanContext { if sc.trace == nil { sc.trace = newTrace() } - sc.trace.tags = ctx.Tags() // +checklocksignore - Initialization time, not shared yet. - sc.trace.propagatingTags = ctx.PropagatingTags() // +checklocksignore - Initialization time, not shared yet. + sc.trace.tags = ctx.Tags() // +checklocksignore - Initialization time, not shared yet. + sc.trace.propagatingTags = ctx.PropagatingTags() // +checklocksignore - Initialization time, not shared yet. + if dm, ok := sc.trace.propagatingTags[keyDecisionMaker]; ok { // +checklocksignore - Initialization time, not shared yet. + sc.trace.dm = parseDecisionMaker(dm) // +checklocksignore - Initialization time, not shared yet. + } return &sc } @@ -464,6 +467,10 @@ type trace struct { // specifies if the sampling priority can be altered // +checklocks:mu locked bool + // dm is the numeric form of _dd.p.dm for v1 protocol encoding. + // It is the absolute value of the parsed integer (e.g., "-4" → 4). + // +checklocks:mu + dm uint32 // samplingDecision indicates whether to send the trace to the agent. samplingDecision samplingDecision // +checkatomic @@ -600,7 +607,7 @@ func (t *trace) setSamplingPriorityLockedWithForce(p int, sampler samplernames.S } } if p <= 0 && existed { - delete(t.propagatingTags, keyDecisionMaker) + t.unsetPropagatingTagLocked(keyDecisionMaker) } return updatedPriority diff --git a/ddtrace/tracer/spancontext_test.go b/ddtrace/tracer/spancontext_test.go index f698caaee44..c50fc1d72ac 100644 --- a/ddtrace/tracer/spancontext_test.go +++ b/ddtrace/tracer/spancontext_test.go @@ -1337,3 +1337,33 @@ func BenchmarkUpdateTracerGitMetadataTags(b *testing.B) { } }) } + +// genericCtxWithDM is a minimal ddtrace.SpanContext + spanContextV1Adapter that +// carries a _dd.p.dm propagating tag, simulating a context arriving from an +// external (non-native) integration. +type genericCtxWithDM struct { + propagatingTags map[string]string +} + +func (g *genericCtxWithDM) SpanID() uint64 { return 1 } +func (g *genericCtxWithDM) TraceID() string { return "1" } +func (g *genericCtxWithDM) TraceIDBytes() [16]byte { var b [16]byte; b[15] = 1; return b } +func (g *genericCtxWithDM) TraceIDLower() uint64 { return 1 } +func (g *genericCtxWithDM) ForeachBaggageItem(_ func(k, v string) bool) {} +func (g *genericCtxWithDM) SamplingDecision() uint32 { return uint32(decisionKeep) } +func (g *genericCtxWithDM) Priority() *float64 { p := 1.0; return &p } +func (g *genericCtxWithDM) Origin() string { return "" } +func (g *genericCtxWithDM) PropagatingTags() map[string]string { return g.propagatingTags } +func (g *genericCtxWithDM) Tags() map[string]string { return nil } + +// TestFromGenericCtxDecisionMakerCached verifies that FromGenericCtx populates +// t.dm so that trace.decisionMaker() returns the correct sampling mechanism for +// traces arriving through generic (non-native) context adapters. +func TestFromGenericCtxDecisionMakerCached(t *testing.T) { + ctx := &genericCtxWithDM{ + propagatingTags: map[string]string{keyDecisionMaker: "-4"}, + } + sc := FromGenericCtx(ctx) + require.NotNil(t, sc.trace) + assert.Equal(t, uint32(4), sc.trace.decisionMaker()) +} From 97a2bf9d497f6f716b92cdfa68b59e985f12787f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Wed, 25 Mar 2026 06:14:28 -0400 Subject: [PATCH 24/58] fix(ddtrace/tracer): check `DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED` once, use a global `bool` (#4548) Stop checking environment variable `DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED` on span start. Checks it once, use a global `bool`. It'd be better to load it on `newConfig` and make the value available, but this one happens in `newSpanContext`, which modification will cause a lot of changes in multiple files, including tests. Complete #1905 work using the current benchmarking platform. - [x] Changed code has unit tests for its functionality at or near 100% coverage. - [x] There is a benchmark for any new code, or changes to existing code. - [x] New code is free of linting errors. You can check this by running `make lint` locally. - [x] New code doesn't break existing tests. You can check this by running `make test` locally. - [x] Add an appropriate team label so this PR gets put in the right place for the release notes. Unsure? Have a question? Request a review! Co-authored-by: kakkoyun Co-authored-by: kemal.akkoyun --- ddtrace/tracer/context_test.go | 3 ++- ddtrace/tracer/option.go | 2 ++ ddtrace/tracer/span_test.go | 12 ++++++++---- ddtrace/tracer/spancontext.go | 13 ++++++++++++- ddtrace/tracer/textmap_test.go | 3 ++- ddtrace/tracer/tracer_test.go | 10 ++++++---- internal/config/config.go | 9 +++++++++ 7 files changed, 41 insertions(+), 11 deletions(-) diff --git a/ddtrace/tracer/context_test.go b/ddtrace/tracer/context_test.go index 7f2fab71cc8..a6b4ad952f2 100644 --- a/ddtrace/tracer/context_test.go +++ b/ddtrace/tracer/context_test.go @@ -150,7 +150,8 @@ func Test128(t *testing.T) { defer stop() t.Run("disable 128 bit trace ids", func(t *testing.T) { - t.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") + old := traceID128BitEnabled.Swap(false) + defer func(v bool) { traceID128BitEnabled.Store(v) }(old) span, _ := StartSpanFromContext(context.Background(), "http.request") assert.NotZero(t, span.Context().TraceID()) w3cCtx := span.Context() diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 43e6d6fb305..53d57477cee 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -500,6 +500,8 @@ func newConfig(opts ...StartOption) (*config, error) { c.llmobs.AgentFeatures = llmobsconfig.AgentFeatures{ EVPProxyV2: af.evpProxyV2, } + // Set global 128-bits trace ID generation variable + traceID128BitEnabled.Store(c.internalConfig.TraceID128BitEnabled()) return c, nil } diff --git a/ddtrace/tracer/span_test.go b/ddtrace/tracer/span_test.go index 3291b0dfe5b..a2e54bd1f1b 100644 --- a/ddtrace/tracer/span_test.go +++ b/ddtrace/tracer/span_test.go @@ -1316,11 +1316,12 @@ func TestSpanLog(t *testing.T) { t.Run("128-bit-logging-only", func(t *testing.T) { // Logging 128-bit trace ids is enabled, but 128bit format is not present in // the span. So only the lower 64 bits should be logged in decimal form. - t.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") assert := assert.New(t) tracer, _, _, stop, err := startTestTracer(t, WithService("tracer.test"), WithEnv("testenv")) assert.Nil(err) defer stop() + old := traceID128BitEnabled.Swap(false) + defer func(v bool) { traceID128BitEnabled.Store(v) }(old) span := tracer.StartSpan("test.request") span.traceID = 12345678 span.spanID = 87654321 @@ -1332,7 +1333,8 @@ func TestSpanLog(t *testing.T) { t.Run("128-bit-logging-with-generation", func(t *testing.T) { // Logging 128-bit trace ids is enabled, and a 128-bit trace id, so // a quoted 32 byte hex string should be printed for the dd.trace_id. - t.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "true") + old := traceID128BitEnabled.Swap(true) + defer func(v bool) { traceID128BitEnabled.Store(v) }(old) t.Setenv("DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED", "true") assert := assert.New(t) tracer, _, _, stop, err := startTestTracer(t, WithService("tracer.test"), WithEnv("testenv")) @@ -1350,7 +1352,8 @@ func TestSpanLog(t *testing.T) { t.Run("128-bit-logging-with-small-upper-bits", func(t *testing.T) { // Logging 128-bit trace ids is enabled, and a 128-bit trace id, so // a quoted 32 byte hex string should be printed for the dd.trace_id. - t.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") + old := traceID128BitEnabled.Swap(false) + defer func(v bool) { traceID128BitEnabled.Store(v) }(old) assert := assert.New(t) tracer, _, _, stop, err := startTestTracer(t, WithService("tracer.test"), WithEnv("testenv")) assert.Nil(err) @@ -1366,11 +1369,12 @@ func TestSpanLog(t *testing.T) { t.Run("128-bit-logging-with-empty-upper-bits", func(t *testing.T) { // Logging 128-bit trace ids is enabled, but the upper 64 bits // are empty, so the dd.trace_id should be printed as raw digits (not hex). - t.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") assert := assert.New(t) tracer, _, _, stop, err := startTestTracer(t, WithService("tracer.test"), WithEnv("testenv")) assert.Nil(err) defer stop() + old := traceID128BitEnabled.Swap(false) + defer func(v bool) { traceID128BitEnabled.Store(v) }(old) span := tracer.StartSpan("test.request", WithSpanID(87654321)) span.Finish() assert.False(span.context.traceID.HasUpper()) // it should not have generated upper bits diff --git a/ddtrace/tracer/spancontext.go b/ddtrace/tracer/spancontext.go index 4e0cdd184e0..c35818bc85a 100644 --- a/ddtrace/tracer/spancontext.go +++ b/ddtrace/tracer/spancontext.go @@ -28,6 +28,17 @@ import ( const TraceIDZero string = "00000000000000000000000000000000" +// traceID128BitEnabled caches DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED +// at init time so that newSpanContext avoids calling BoolEnv on every span. +// It is re-set by newConfig on every tracer start so that restarts pick up +// any changed value. The init here ensures it is correct even when using +// mocktracer (which does not call newConfig). +var traceID128BitEnabled atomic.Bool + +func init() { + traceID128BitEnabled.Store(sharedinternal.BoolEnv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", true)) +} + var _ ddtrace.SpanContext = (*SpanContext)(nil) // traceID in big endian, i.e. @@ -228,7 +239,7 @@ func newSpanContext(span *Span, parent *SpanContext) *SpanContext { context.setBaggageItem(k, v) return true }) - } else if sharedinternal.BoolEnv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", true) { + } else if traceID128BitEnabled.Load() { // add 128 bit trace id, if enabled, formatted as big-endian: // <32-bit unix seconds> <32 bits of zero> <64 random bits> id128 := time.Duration(span.start) / time.Second diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index be9aff7d774..5aa49e453cf 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -561,7 +561,8 @@ func TestTextMapPropagator(t *testing.T) { }) t.Run("InjectExtract", func(t *testing.T) { - t.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "true") + old := traceID128BitEnabled.Swap(true) + defer func(v bool) { traceID128BitEnabled.Store(v) }(old) t.Setenv(headerPropagationStyleExtract, "datadog") t.Setenv(headerPropagationStyleInject, "datadog") propagator := NewPropagator(&PropagatorConfig{ diff --git a/ddtrace/tracer/tracer_test.go b/ddtrace/tracer/tracer_test.go index d01014290da..e344f57eeb8 100644 --- a/ddtrace/tracer/tracer_test.go +++ b/ddtrace/tracer/tracer_test.go @@ -809,7 +809,8 @@ func TestTracerStartSpanOptions128(t *testing.T) { defer setGlobalTracer(&NoopTracer{}) t.Run("64-bit-trace-id", func(t *testing.T) { assert := assert.New(t) - t.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") + old := traceID128BitEnabled.Swap(false) + defer func(v bool) { traceID128BitEnabled.Store(v) }(old) opts := []StartSpanOption{ WithSpanID(987654), } @@ -1251,15 +1252,16 @@ func TestNewSpanChild(t *testing.T) { func testNewSpanChild(t *testing.T, is128 bool) { t.Run(fmt.Sprintf("TestNewChildSpan(is128=%t)", is128), func(t *testing.T) { - if !is128 { - t.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") - } assert := assert.New(t) // the tracer must create child spans tracer, err := newTracer(withTransport(newDefaultTransport())) setGlobalTracer(tracer) defer tracer.Stop() + if !is128 { + old := traceID128BitEnabled.Swap(false) + defer func(v bool) { traceID128BitEnabled.Store(v) }(old) + } assert.Nil(err) parent := tracer.newRootSpan("pylons.request", "pylons", "/") child := tracer.newChildSpan("redis.command", parent) diff --git a/internal/config/config.go b/internal/config/config.go index 47144462ad6..42381318be1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -120,6 +120,8 @@ type Config struct { // otlpHeaders holds the resolved OTLP trace headers from // OTEL_EXPORTER_OTLP_TRACES_HEADERS plus Content-Type: application/x-protobuf. otlpHeaders map[string]string + // traceID128BitEnabled controls if trace IDs are generated as 128-bits or 64-bits. + traceID128BitEnabled bool } // loadConfig initializes and returns a new config by reading from all configured sources. @@ -171,6 +173,7 @@ func loadConfig() *Config { } cfg.otlpTraceURL = resolveOTLPTraceURL(cfg.agentURL, p.GetString("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "")) cfg.otlpHeaders = buildOTLPHeaders(p.GetMap("OTEL_EXPORTER_OTLP_TRACES_HEADERS", nil, internal.OtelTagsDelimeter)) + cfg.traceID128BitEnabled = p.GetBool("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", true) // Parse feature flags from DD_TRACE_FEATURES as a set cfg.featureFlags = make(map[string]struct{}) @@ -744,3 +747,9 @@ func (c *Config) OTLPHeaders() map[string]string { defer c.mu.RUnlock() return maps.Clone(c.otlpHeaders) } + +func (c *Config) TraceID128BitEnabled() bool { + c.mu.RLock() + defer c.mu.RUnlock() + return c.traceID128BitEnabled +} From ffbea4edbd4bdbe40dbb103658d366f6c6649701 Mon Sep 17 00:00:00 2001 From: Moe Zein Date: Wed, 25 Mar 2026 09:48:55 -0400 Subject: [PATCH 25/58] chore: disable automated dependency updater config [incident-51602] Signed-off-by: Moe Zein --- .github/{dependabot.yml => dependabot.yml.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{dependabot.yml => dependabot.yml.disabled} (100%) diff --git a/.github/dependabot.yml b/.github/dependabot.yml.disabled similarity index 100% rename from .github/dependabot.yml rename to .github/dependabot.yml.disabled From 78e160f08d75688d901fa903a53b7c01ddf81512 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler <46911781+mtoffl01@users.noreply.github.com> Date: Wed, 25 Mar 2026 11:38:37 -0400 Subject: [PATCH 26/58] Apply suggestion from @genesor Co-authored-by: Benjamin De Bernardi --- internal/config/provider/provider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/config/provider/provider.go b/internal/config/provider/provider.go index a7429fcd448..fbda09d8c8f 100644 --- a/internal/config/provider/provider.go +++ b/internal/config/provider/provider.go @@ -154,7 +154,7 @@ func (p *Provider) GetFloatWithValidator(key string, def float64, validate func( }) } -// / IsSet returns true if any configuration source provides a non-empty value for the key. +// IsSet returns true if any configuration source provides a non-empty value for the key. // // TODO: populate an isSet field on the Provider at the time of iterating over // sources instead of re-querying them here. From f0c6f00b688d5e8e77b7fa5f0041a37363ab303d Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Wed, 25 Mar 2026 11:52:59 -0400 Subject: [PATCH 27/58] address PR review comments --- internal/config/config_helpers.go | 13 +++++++++++-- internal/config/config_helpers_test.go | 25 +++++++++++++++++++++---- internal/config/config_test.go | 2 -- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/internal/config/config_helpers.go b/internal/config/config_helpers.go index 0b4c9469f10..6fe92f58af7 100644 --- a/internal/config/config_helpers.go +++ b/internal/config/config_helpers.go @@ -143,10 +143,19 @@ func detectUDSURL() *url.URL { } } -// resolveOTLPTraceURL resolves the OTLP trace endpoint from OTEL_EXPORTER_OTLP_TRACES_ENDPOINT if set, else agentURL host + default OTLP port 4318 + /v1/traces +// resolveOTLPTraceURL resolves the OTLP trace endpoint from OTEL_EXPORTER_OTLP_TRACES_ENDPOINT if set, else agentURL host + default OTLP port 4318 + /v1/traces. +// When the user-provided endpoint is set, it is validated: it must be a parseable URL with an http or https scheme. +// If validation fails, the default endpoint is used instead. func resolveOTLPTraceURL(rawAgentURL *url.URL, otlpTracesEndpoint string) string { if otlpTracesEndpoint != "" { - return otlpTracesEndpoint + u, err := url.Parse(otlpTracesEndpoint) + if err != nil { + log.Warn("Failed to parse OTEL_EXPORTER_OTLP_TRACES_ENDPOINT %q: %s. Falling back to default.", otlpTracesEndpoint, err.Error()) + } else if u.Scheme != URLSchemeHTTP && u.Scheme != URLSchemeHTTPS { + log.Warn("Unsupported scheme %q in OTEL_EXPORTER_OTLP_TRACES_ENDPOINT %q. Must be %s or %s. Falling back to default.", u.Scheme, otlpTracesEndpoint, URLSchemeHTTP, URLSchemeHTTPS) + } else { + return otlpTracesEndpoint + } } host := internal.DefaultAgentHostname if rawAgentURL != nil { diff --git a/internal/config/config_helpers_test.go b/internal/config/config_helpers_test.go index 8f43dde8786..5ece1188487 100644 --- a/internal/config/config_helpers_test.go +++ b/internal/config/config_helpers_test.go @@ -14,25 +14,42 @@ import ( func TestResolveOTLPTraceURL(t *testing.T) { httpAgent := &url.URL{Scheme: "http", Host: "myhost:8126"} + defaultWithAgent := "http://myhost:4318/v1/traces" + defaultLocalhost := "http://localhost:4318/v1/traces" - t.Run("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT used when set", func(t *testing.T) { + t.Run("valid http endpoint used when set", func(t *testing.T) { got := resolveOTLPTraceURL(httpAgent, "http://traces-collector:4318/v1/traces") assert.Equal(t, "http://traces-collector:4318/v1/traces", got) }) + t.Run("valid https endpoint used when set", func(t *testing.T) { + got := resolveOTLPTraceURL(httpAgent, "https://traces-collector:4318/v1/traces") + assert.Equal(t, "https://traces-collector:4318/v1/traces", got) + }) + + t.Run("unsupported scheme falls back to default", func(t *testing.T) { + got := resolveOTLPTraceURL(httpAgent, "grpc://traces-collector:4317") + assert.Equal(t, defaultWithAgent, got) + }) + + t.Run("missing scheme falls back to default", func(t *testing.T) { + got := resolveOTLPTraceURL(httpAgent, "traces-collector:4318/v1/traces") + assert.Equal(t, defaultWithAgent, got) + }) + t.Run("default uses agent host with OTLP port", func(t *testing.T) { got := resolveOTLPTraceURL(httpAgent, "") - assert.Equal(t, "http://myhost:4318/v1/traces", got) + assert.Equal(t, defaultWithAgent, got) }) t.Run("default with nil agent URL uses localhost", func(t *testing.T) { got := resolveOTLPTraceURL(nil, "") - assert.Equal(t, "http://localhost:4318/v1/traces", got) + assert.Equal(t, defaultLocalhost, got) }) t.Run("default with unix socket agent uses localhost", func(t *testing.T) { unixAgent := &url.URL{Scheme: "unix", Path: "/var/run/datadog/apm.socket"} got := resolveOTLPTraceURL(unixAgent, "") - assert.Equal(t, "http://localhost:4318/v1/traces", got) + assert.Equal(t, defaultLocalhost, got) }) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index f1f667e6f09..d2921bc3ab6 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -6,7 +6,6 @@ package config import ( - "fmt" "net/url" "reflect" "sort" @@ -522,7 +521,6 @@ func TestOTLPHeaders(t *testing.T) { require.NotNil(t, cfg) headers := cfg.OTLPHeaders() - fmt.Println(t, headers) assert.Equal(t, "secret", headers["api-key"]) assert.Equal(t, "value", headers["x-custom"]) assert.Equal(t, OTLPContentTypeHeader, headers["Content-Type"]) From 996c4d7e3fae00005922a37f0ee601eaed4b883b Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Wed, 25 Mar 2026 13:40:59 -0400 Subject: [PATCH 28/58] rename transport to ddtransport: be clear --- ddtrace/tracer/civisibility_transport.go | 4 +-- ddtrace/tracer/civisibility_writer.go | 4 +-- ddtrace/tracer/civisibility_writer_test.go | 2 +- ddtrace/tracer/log.go | 4 +-- ddtrace/tracer/option.go | 14 ++++----- ddtrace/tracer/option_test.go | 4 +-- ddtrace/tracer/span_test.go | 4 +-- ddtrace/tracer/stats.go | 2 +- ddtrace/tracer/stats_test.go | 12 ++++---- ddtrace/tracer/tracer_test.go | 13 +------- ddtrace/tracer/transport.go | 35 ++-------------------- ddtrace/tracer/transport_test.go | 26 ++++++++-------- ddtrace/tracer/writer.go | 2 +- ddtrace/tracer/writer_test.go | 6 ++-- 14 files changed, 46 insertions(+), 86 deletions(-) diff --git a/ddtrace/tracer/civisibility_transport.go b/ddtrace/tracer/civisibility_transport.go index 234e7bf2de1..b219c5ac402 100644 --- a/ddtrace/tracer/civisibility_transport.go +++ b/ddtrace/tracer/civisibility_transport.go @@ -33,8 +33,8 @@ const ( EvpProxyPath = "evp_proxy/v2" // Path for EVP proxy. ) -// Ensure that civisibilityTransport implements the transport interface. -var _ transport = (*ciVisibilityTransport)(nil) +// Ensure that ciVisibilityTransport implements the ddTransport interface. +var _ ddTransport = (*ciVisibilityTransport)(nil) // ciVisibilityTransport is a structure that handles sending CI Visibility payloads // to the Datadog endpoint, either in agentless mode or through the EVP proxy. diff --git a/ddtrace/tracer/civisibility_writer.go b/ddtrace/tracer/civisibility_writer.go index 584e970cc28..c7967e1c754 100644 --- a/ddtrace/tracer/civisibility_writer.go +++ b/ddtrace/tracer/civisibility_writer.go @@ -108,7 +108,7 @@ func (w *ciVisibilityTraceWriter) flush() { var err error requestCompressedType := telemetry.UncompressedRequestCompressedType - if ciTransport, ok := w.config.transport.(*ciVisibilityTransport); ok && ciTransport.agentless { + if ciTransport, ok := w.config.ddTransport.(*ciVisibilityTransport); ok && ciTransport.agentless { requestCompressedType = telemetry.CompressedRequestCompressedType } telemetry.EndpointPayloadRequests(telemetry.TestCycleEndpointType, requestCompressedType) @@ -117,7 +117,7 @@ func (w *ciVisibilityTraceWriter) flush() { stats := p.stats() size, count = stats.size, stats.itemCount log.Debug("ciVisibilityTraceWriter: sending payload: size: %d events: %d\n", size, count) - _, err = w.config.transport.send(p.payload) + _, err = w.config.ddTransport.send(p.payload) if err == nil { log.Debug("ciVisibilityTraceWriter: sent events after %d attempts", attempt+1) return diff --git a/ddtrace/tracer/civisibility_writer_test.go b/ddtrace/tracer/civisibility_writer_test.go index e2bb0eda165..43bb14efc8c 100644 --- a/ddtrace/tracer/civisibility_writer_test.go +++ b/ddtrace/tracer/civisibility_writer_test.go @@ -91,7 +91,7 @@ func TestCiVisibilityTraceWriterFlushRetries(t *testing.T) { assert: assert, } c, err := newTestConfig(func(c *config) { - c.transport = p + c.ddTransport = p c.sendRetries = test.configRetries c.internalConfig.SetRetryInterval(test.retryInterval, internalconfig.OriginCode) }) diff --git a/ddtrace/tracer/log.go b/ddtrace/tracer/log.go index fdd989d9ea2..be674fc16a4 100644 --- a/ddtrace/tracer/log.go +++ b/ddtrace/tracer/log.go @@ -122,7 +122,7 @@ func logStartup(t *tracer) { if srcURL := t.config.internalConfig.RawAgentURL(); srcURL != nil && srcURL.Scheme == "unix" { agentURL = srcURL.String() } else { - agentURL = t.config.transport.endpoint() + agentURL = t.config.ddTransport.endpoint() } info := startupInfo{ Date: time.Now().Format(time.RFC3339), @@ -170,7 +170,7 @@ func logStartup(t *tracer) { info.SampleRateLimit = fmt.Sprintf("%v", limit) } if !t.config.internalConfig.LogToStdout() { - if err := checkEndpoint(t.config.httpClient, t.config.transport.endpoint(), t.config.internalConfig.TraceProtocol()); err != nil { + if err := checkEndpoint(t.config.httpClient, t.config.ddTransport.endpoint(), t.config.internalConfig.TraceProtocol()); err != nil { info.AgentError = err.Error() log.Warn("DIAGNOSTICS Unable to reach agent intake: %s", err.Error()) } diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 53d57477cee..1dd4d8c75c3 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -174,8 +174,8 @@ type config struct { // all spans. globalTags dynamicConfig[map[string]any] - // transport specifies the Transport interface which will be used to send data to the agent. - transport transport + // ddTransport specifies the Datadog transport used to send msgpack traces and stats to the agent. + ddTransport ddTransport // httpClientTimeout specifies the timeout for the HTTP client. httpClientTimeout time.Duration @@ -417,10 +417,10 @@ func newConfig(opts ...StartOption) (*config, error) { } else { globalconfig.SetServiceName(c.internalConfig.ServiceName()) } - if c.transport == nil { + if c.ddTransport == nil { agentURL := c.internalConfig.AgentURL().String() traceURL, headers := resolveTraceTransport(c.internalConfig) - c.transport = newHTTPTransport(traceURL, agentURL+statsAPIPath, c.httpClient, headers) + c.ddTransport = newHTTPTransport(traceURL, agentURL+statsAPIPath, c.httpClient, headers) } if c.propagator == nil { envKey := "DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH" @@ -448,7 +448,7 @@ func newConfig(opts ...StartOption) (*config, error) { c.httpClientTimeout = time.Second * 45 // Increase timeout up to 45 seconds (same as other tracers in CIVis mode) c.internalConfig.SetLogStartup(false, internalconfig.OriginCalculated) // If we are in CI Visibility mode we don't want to log the startup to stdout to avoid polluting the output ciTransport := newCiVisibilityTransport(c) // Create a default CI Visibility Transport - c.transport = ciTransport // Replace the default transport with the CI Visibility transport + c.ddTransport = ciTransport // Replace the default transport with the CI Visibility transport c.ciVisibilityAgentless = ciTransport.agentless c.ciVisibilityNoopTracer = internal.BoolEnv(constants.CIVisibilityUseNoopTracer, false) } @@ -461,7 +461,7 @@ func newConfig(opts ...StartOption) (*config, error) { // If the agent doesn't support the v1 protocol, downgrade to v0.4 if c.internalConfig.TraceProtocol() == traceProtocolV1 && !af.v1ProtocolAvailable { c.internalConfig.SetTraceProtocol(traceProtocolV04, internalconfig.OriginCalculated) - if t, ok := c.transport.(*httpTransport); ok { + if t, ok := c.ddTransport.(*httpTransport); ok { t.traceURL = agentURL.String() + tracesAPIPath } } @@ -1455,7 +1455,7 @@ func WithTestDefaults(statsdClient any) StartOption { statsdClient = &statsd.NoOpClientDirect{} } c.statsdClient = statsdClient.(internal.StatsdClient) - c.transport = newDummyTransport() + c.ddTransport = newDummyTransport() } } diff --git a/ddtrace/tracer/option_test.go b/ddtrace/tracer/option_test.go index 9e236ab926f..68432d94f1a 100644 --- a/ddtrace/tracer/option_test.go +++ b/ddtrace/tracer/option_test.go @@ -38,9 +38,9 @@ import ( "github.com/DataDog/dd-trace-go/v2/internal/version" ) -func withTransport(t transport) StartOption { +func withTransport(t ddTransport) StartOption { return func(c *config) { - c.transport = t + c.ddTransport = t } } diff --git a/ddtrace/tracer/span_test.go b/ddtrace/tracer/span_test.go index a2e54bd1f1b..d73d0ada03b 100644 --- a/ddtrace/tracer/span_test.go +++ b/ddtrace/tracer/span_test.go @@ -1763,7 +1763,7 @@ func TestStatsAfterFinish(t *testing.T) { setGlobalTracer(tracer) transport := newDummyTransport() - tracer.config.transport = transport + tracer.config.ddTransport = transport af := tracer.config.agent.load() af.Stats = true af.DropP0s = true @@ -1803,7 +1803,7 @@ func TestStatsAfterFinish(t *testing.T) { setGlobalTracer(tracer) transport := newDummyTransport() - tracer.config.transport = transport + tracer.config.ddTransport = transport af2 := tracer.config.agent.load() af2.Stats = true af2.DropP0s = true diff --git a/ddtrace/tracer/stats.go b/ddtrace/tracer/stats.go index e18fcf6d8ae..38a23d1ca0c 100644 --- a/ddtrace/tracer/stats.go +++ b/ddtrace/tracer/stats.go @@ -255,7 +255,7 @@ func (c *concentrator) flushAndSend(timenow time.Time, includeCurrent bool) { flushedBuckets += len(csp.Stats) var err error for attempt := 0; attempt <= c.cfg.sendRetries; attempt++ { - err = c.cfg.transport.sendStats(csp, obfVersion) + err = c.cfg.ddTransport.sendStats(csp, obfVersion) if err == nil { break } diff --git a/ddtrace/tracer/stats_test.go b/ddtrace/tracer/stats_test.go index ac5563f2d49..52bea73c8d7 100644 --- a/ddtrace/tracer/stats_test.go +++ b/ddtrace/tracer/stats_test.go @@ -36,20 +36,20 @@ func TestAlignTs(t *testing.T) { assert.Equal(t, got, want) } -func newTestConfigWithTransportAndEnv(t *testing.T, transport transport, env string) *config { +func newTestConfigWithTransportAndEnv(t *testing.T, transport ddTransport, env string) *config { assert := assert.New(t) cfg, err := newTestConfig(withNoopInfoHTTPClient(), func(c *config) { - c.transport = transport + c.ddTransport = transport c.internalConfig.SetEnv(env, internalconfig.OriginCode) }) assert.NoError(err) return cfg } -func newTestConfigWithTransport(t *testing.T, transport transport) *config { +func newTestConfigWithTransport(t *testing.T, transport ddTransport) *config { assert := assert.New(t) cfg, err := newTestConfig(withNoopInfoHTTPClient(), func(c *config) { - c.transport = transport + c.ddTransport = transport }) assert.NoError(err) return cfg @@ -289,7 +289,7 @@ func TestConcentratorDefaultEnv(t *testing.T) { t.Run("uses-agent-default-env-when-no-tracer-env", func(t *testing.T) { cfg, err := newTestConfig(func(c *config) { - c.transport = newDummyTransport() + c.ddTransport = newDummyTransport() }) assert.NoError(err) af := cfg.agent.load() @@ -457,7 +457,7 @@ func TestStatsFlushRetries(t *testing.T) { t.Run(name, func(t *testing.T) { p := &failingStatsTransport{failCount: test.failCount} cfg, err := newTestConfig(func(c *config) { - c.transport = p + c.ddTransport = p c.sendRetries = test.configRetries c.internalConfig.SetRetryInterval(test.retryInterval, internalconfig.OriginCode) c.internalConfig.SetEnv("someEnv", internalconfig.OriginCode) diff --git a/ddtrace/tracer/tracer_test.go b/ddtrace/tracer/tracer_test.go index e344f57eeb8..48cc47e3952 100644 --- a/ddtrace/tracer/tracer_test.go +++ b/ddtrace/tracer/tracer_test.go @@ -1227,7 +1227,7 @@ func TestTracerNoDebugStack(t *testing.T) { } // newDefaultTransport return a default transport for this tracing client -func newDefaultTransport() transport { +func newDefaultTransport() ddTransport { return newHTTPTransport(defaultURL+tracesAPIPath, defaultURL+statsAPIPath, internal.DefaultHTTPClient(defaultHTTPTimeout, true), datadogHeaders()) } @@ -1470,17 +1470,6 @@ func TestOTLPExportMode(t *testing.T) { assert.True(isAlwaysOn, "expected otelParentBasedAlwaysOnSampler in OTLP export mode") }) - t.Run("default mode uses agentTraceWriter and prioritySampler", func(t *testing.T) { - assert := assert.New(t) - tracer, err := newUnstartedTracer() - assert.NoError(err) - defer tracer.Stop() - _, isAgentWriter := tracer.traceWriter.(*agentTraceWriter) - assert.True(isAgentWriter, "expected agentTraceWriter in default mode") - _, isPriority := tracer.defaultSampler.(*prioritySampler) - assert.True(isPriority, "expected prioritySampler in default mode") - }) - t.Run("OTEL_TRACES_EXPORTER=otlp env var enables OTLP mode", func(t *testing.T) { assert := assert.New(t) t.Setenv("OTEL_TRACES_EXPORTER", "otlp") diff --git a/ddtrace/tracer/transport.go b/ddtrace/tracer/transport.go index 9415ac06e0b..5d8d905cba7 100644 --- a/ddtrace/tracer/transport.go +++ b/ddtrace/tracer/transport.go @@ -44,8 +44,9 @@ const ( statsAPIPath = "/v0.6/stats" ) -// transport is an interface for communicating data to the agent. -type transport interface { +// ddTransport is an interface for communicating data to the Datadog agent +// using Datadog-specific protocols (msgpack traces, stats payloads). +type ddTransport interface { // send sends the msgpack-encoded payload p to the agent using the transport set up. // It returns a non-nil response body when no error occurred. send(p payload) (body io.ReadCloser, err error) @@ -56,13 +57,6 @@ type transport interface { endpoint() string } -// otlpSender is a narrow interface for sending protobuf-encoded OTLP trace payloads. -// It is satisfied by httpTransport and used by otlpTraceWriter to avoid coupling -// the OTLP export path to the full transport interface. -type otlpSender interface { - sendOTLP(data []byte) (body io.ReadCloser, err error) -} - type httpTransport struct { traceURL string // the delivery URL for traces statsURL string // the delivery URL for stats @@ -194,29 +188,6 @@ func (t *httpTransport) send(p payload) (body io.ReadCloser, err error) { return response.Body, nil } -// sendOTLP sends a protobuf-encoded OTLP payload to the configured trace endpoint. -// Content-Length is set automatically by net/http because the body is a *bytes.Reader. -func (t *httpTransport) sendOTLP(p []byte) (body io.ReadCloser, err error) { - req, err := http.NewRequest("POST", t.traceURL, bytes.NewReader(p)) - if err != nil { - return nil, fmt.Errorf("cannot create http request: %s", err) - } - for header, value := range t.headers { - req.Header.Set(header, value) - } - response, err := t.client.Do(req) - // TODO: t.statsd will point to an agent address that is probably not in use; we need to decide what to do with these metrics. - if err != nil { - // reportAPIErrorsMetric(response, err, tracesAPIPath) - return nil, err - } - if code := response.StatusCode; code >= 400 { - // reportAPIErrorsMetric(response, err, tracesAPIPath) - return nil, fmt.Errorf("status code: %d", code) - } - return response.Body, nil -} - func reportAPIErrorsMetric(response *http.Response, err error, endpoint string) { if t, ok := getGlobalTracer().(*tracer); ok { var reason string diff --git a/ddtrace/tracer/transport_test.go b/ddtrace/tracer/transport_test.go index e5b78afc3be..352f77f6374 100644 --- a/ddtrace/tracer/transport_test.go +++ b/ddtrace/tracer/transport_test.go @@ -294,7 +294,7 @@ func TestApiErrorsMetric(t *testing.T) { assert.NoError(err) // We're expecting an error - _, err = trc.config.transport.send(p) + _, err = trc.config.ddTransport.send(p) assert.Error(err) calls := statsdtest.FilterCallsByName(tg.IncrCalls(), "datadog.tracer.api.errors") assert.Len(calls, 1) @@ -316,7 +316,7 @@ func TestApiErrorsMetric(t *testing.T) { p, err := encode(getTestTrace(1, 1)) assert.NoError(err) - _, err = trc.config.transport.send(p) + _, err = trc.config.ddTransport.send(p) assert.Error(err) calls := statsdtest.FilterCallsByName(tg.IncrCalls(), "datadog.tracer.api.errors") @@ -336,7 +336,7 @@ func TestApiErrorsMetric(t *testing.T) { defer trc.Stop() // We're expecting an error - err = trc.config.transport.sendStats(&pb.ClientStatsPayload{}, 1) + err = trc.config.ddTransport.sendStats(&pb.ClientStatsPayload{}, 1) assert.Error(err) calls := statsdtest.FilterCallsByName(tg.IncrCalls(), "datadog.tracer.api.errors") assert.Len(calls, 1) @@ -354,7 +354,7 @@ func TestApiErrorsMetric(t *testing.T) { setGlobalTracer(trc) defer trc.Stop() - err = trc.config.transport.sendStats(&pb.ClientStatsPayload{}, 1) + err = trc.config.ddTransport.sendStats(&pb.ClientStatsPayload{}, 1) assert.Error(err) calls := statsdtest.FilterCallsByName(tg.IncrCalls(), "datadog.tracer.api.errors") @@ -376,7 +376,7 @@ func TestApiErrorsMetric(t *testing.T) { p, err := encode(getTestTrace(1, 1)) assert.NoError(err) - _, err = trc.config.transport.send(p) + _, err = trc.config.ddTransport.send(p) assert.NoError(err) calls := statsdtest.FilterCallsByName(tg.IncrCalls(), "datadog.tracer.api.errors") @@ -410,7 +410,7 @@ func TestWithHTTPClient(t *testing.T) { p, err := encode(getTestTrace(1, 1)) assert.NoError(err) - _, err = trc.config.transport.send(p) + _, err = trc.config.ddTransport.send(p) assert.NoError(err) assert.Len(rt.reqs, 2) assert.Contains(rt.reqs[0].URL.Path, "/info") @@ -448,7 +448,7 @@ func TestWithUDS(t *testing.T) { p, err := encode(getTestTrace(1, 1)) assert.NoError(err) - body, err := trc.config.transport.send(p) + body, err := trc.config.ddTransport.send(p) assert.NoError(err) defer body.Close() // There are 2 requests, but one happens on tracer startup before we wrap the round tripper. @@ -481,7 +481,7 @@ func TestExternalEnvironment(t *testing.T) { p, err := encode(getTestTrace(1, 1)) assert.NoError(err) - _, err = trc.config.transport.send(p) + _, err = trc.config.ddTransport.send(p) assert.NoError(err) assert.True(found) } @@ -510,11 +510,11 @@ func TestDefaultHeaders(t *testing.T) { // Test traces endpoint p, err := encode(getTestTrace(1, 1)) assert.NoError(err) - _, err = trc.config.transport.send(p) + _, err = trc.config.ddTransport.send(p) assert.NoError(err) // Now stats endpoint - err = trc.config.transport.sendStats(&pb.ClientStatsPayload{}, 1) + err = trc.config.ddTransport.sendStats(&pb.ClientStatsPayload{}, 1) assert.NoError(err) } @@ -542,7 +542,7 @@ func TestClientComputedStatsHeader(t *testing.T) { p, err := encode(getTestTrace(1, 1)) assert.NoError(err) - _, err = trc.config.transport.send(p) + _, err = trc.config.ddTransport.send(p) assert.NoError(err) assert.Empty(headerValue, "Datadog-Client-Computed-Stats header should not be set when client_drop_p0s is not supported") }) @@ -570,7 +570,7 @@ func TestClientComputedStatsHeader(t *testing.T) { p, err := encode(getTestTrace(1, 1)) assert.NoError(err) - _, err = trc.config.transport.send(p) + _, err = trc.config.ddTransport.send(p) assert.NoError(err) assert.Empty(headerValue, "Datadog-Client-Computed-Stats header should not be set when stats endpoint is not supported") }) @@ -598,7 +598,7 @@ func TestClientComputedStatsHeader(t *testing.T) { p, err := encode(getTestTrace(1, 1)) assert.NoError(err) - _, err = trc.config.transport.send(p) + _, err = trc.config.ddTransport.send(p) assert.NoError(err) assert.Equal("t", headerValue, "Datadog-Client-Computed-Stats header should be set to 't' when both conditions are met") }) diff --git a/ddtrace/tracer/writer.go b/ddtrace/tracer/writer.go index 08946e8e867..806cf901809 100644 --- a/ddtrace/tracer/writer.go +++ b/ddtrace/tracer/writer.go @@ -139,7 +139,7 @@ func (h *agentTraceWriter) flush() { for attempt := 0; attempt <= h.config.sendRetries; attempt++ { log.Debug("Attempt to send payload: size: %d traces: %d\n", stats.size, stats.itemCount) var rc io.ReadCloser - rc, err = h.config.transport.send(p) + rc, err = h.config.ddTransport.send(p) if err == nil { log.Debug("sent traces after %d attempts", attempt+1) h.statsd.Count("datadog.tracer.flush_bytes", int64(stats.size), nil, 1) diff --git a/ddtrace/tracer/writer_test.go b/ddtrace/tracer/writer_test.go index c1ee016f2ed..1f85415583c 100644 --- a/ddtrace/tracer/writer_test.go +++ b/ddtrace/tracer/writer_test.go @@ -416,7 +416,7 @@ func TestTraceWriterFlushRetries(t *testing.T) { assert: assert, } c, err := newTestConfig(func(c *config) { - c.transport = p + c.ddTransport = p c.sendRetries = test.configRetries c.internalConfig.SetRetryInterval(test.retryInterval, internalconfig.OriginCode) }) @@ -818,7 +818,7 @@ func TestAgentWriterFlushSizeMetrics(t *testing.T) { cfg, err := newTestConfig( withStatsdClient(&tg), func(c *config) { - c.transport = &simpleTransport{} + c.ddTransport = &simpleTransport{} }, ) require.NoError(t, err) @@ -862,7 +862,7 @@ func TestAgentWriterV1FlushPayloadRecycling(t *testing.T) { withStatsdClient(&tg), func(c *config) { c.internalConfig.SetTraceProtocol(traceProtocolV1, internalconfig.OriginCode) - c.transport = &simpleTransport{} + c.ddTransport = &simpleTransport{} }, ) require.NoError(t, err) From 697c2bab50823129e3039b2cf64f14c7f0ec0029 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Wed, 25 Mar 2026 13:41:25 -0400 Subject: [PATCH 29/58] remove otlpSender; make otlpTraceWriter handle its own http client --- ddtrace/tracer/otlp_writer.go | 47 +++++---- ddtrace/tracer/otlp_writer_test.go | 154 ++++++++++++++++------------- 2 files changed, 113 insertions(+), 88 deletions(-) diff --git a/ddtrace/tracer/otlp_writer.go b/ddtrace/tracer/otlp_writer.go index a40e81f0fc4..d1b3ea1bfbf 100644 --- a/ddtrace/tracer/otlp_writer.go +++ b/ddtrace/tracer/otlp_writer.go @@ -6,8 +6,9 @@ package tracer import ( + "bytes" "fmt" - "io" + "net/http" "slices" "sync" "time" @@ -24,7 +25,9 @@ var _ traceWriter = (*otlpTraceWriter)(nil) type otlpTraceWriter struct { config *config - sender otlpSender + client *http.Client + traceURL string + headers map[string]string mu locking.Mutex resource *otlpresource.Resource scope *otlpcommon.InstrumentationScope @@ -33,23 +36,12 @@ type otlpTraceWriter struct { wg sync.WaitGroup } -// noopOTLPSender is a fallback sender that drops all payloads. Used when the -// configured transport does not implement otlpSender. -type noopOTLPSender struct{} - -func (noopOTLPSender) sendOTLP([]byte) (io.ReadCloser, error) { - return nil, fmt.Errorf("OTLP sending is not supported by the configured transport") -} - func newOTLPTraceWriter(c *config) *otlpTraceWriter { - sender, ok := c.transport.(otlpSender) - if !ok { - log.Error("OTLP trace writer: transport %T does not implement otlpSender; traces will be dropped", c.transport) - sender = noopOTLPSender{} - } return &otlpTraceWriter{ config: c, - sender: sender, + client: c.httpClient, + traceURL: c.internalConfig.OTLPTraceURL(), + headers: c.internalConfig.OTLPHeaders(), resource: buildResource(c.internalConfig), scope: &otlpcommon.InstrumentationScope{Name: "dd-trace-go"}, spans: make([]*otlptrace.Span, 0), @@ -70,6 +62,26 @@ func (w *otlpTraceWriter) add(spanList []*Span) { } } +// send posts a protobuf-encoded OTLP payload to the configured collector endpoint. +func (w *otlpTraceWriter) send(data []byte) error { + req, err := http.NewRequest("POST", w.traceURL, bytes.NewReader(data)) + if err != nil { + return fmt.Errorf("cannot create http request: %s", err) + } + for header, value := range w.headers { + req.Header.Set(header, value) + } + resp, err := w.client.Do(req) + if err != nil { + return err + } + resp.Body.Close() + if code := resp.StatusCode; code >= 400 { + return fmt.Errorf("HTTP %d: %s", code, http.StatusText(code)) + } + return nil +} + func (w *otlpTraceWriter) flush() { w.mu.Lock() if len(w.spans) == 0 { @@ -113,7 +125,7 @@ func (w *otlpTraceWriter) flush() { var sendErr error for attempt := 0; attempt <= w.config.sendRetries; attempt++ { log.Debug("OTLP: attempt %d to send payload: %d bytes, %d spans", attempt+1, len(b), spanCount) - _, sendErr = w.sender.sendOTLP(b) + sendErr = w.send(b) if sendErr == nil { log.Debug("OTLP: sent traces after %d attempts", attempt+1) return @@ -126,7 +138,6 @@ func (w *otlpTraceWriter) flush() { } func (w *otlpTraceWriter) stop() { - // TODO: agentTraceWriter reports datadog.tracer.flush_triggered to its statsd client here; do we want to do the same here? w.flush() w.wg.Wait() } diff --git a/ddtrace/tracer/otlp_writer_test.go b/ddtrace/tracer/otlp_writer_test.go index 16be3feb473..c97f5415c3d 100644 --- a/ddtrace/tracer/otlp_writer_test.go +++ b/ddtrace/tracer/otlp_writer_test.go @@ -6,9 +6,10 @@ package tracer import ( - "errors" "fmt" "io" + "net/http" + "net/http/httptest" "sync" "sync/atomic" "testing" @@ -22,49 +23,56 @@ import ( "google.golang.org/protobuf/proto" ) -// mockOTLPSender records sendOTLP calls for test assertions. -type mockOTLPSender struct { - mu sync.Mutex - calls [][]byte - failCount int - sendAttempts int +// testOTLPServer is a test HTTP server that captures OTLP payloads. +type testOTLPServer struct { + *httptest.Server + mu sync.Mutex + payloads [][]byte + // failCount controls how many requests return 500 before succeeding. + failCount int32 } -func (m *mockOTLPSender) sendOTLP(data []byte) (io.ReadCloser, error) { - m.mu.Lock() - defer m.mu.Unlock() - m.sendAttempts++ - if m.failCount > 0 { - m.failCount-- - return nil, errors.New("send failed") - } - cp := make([]byte, len(data)) - copy(cp, data) - m.calls = append(m.calls, cp) - return nil, nil +func newTestOTLPServer() *testOTLPServer { + s := &testOTLPServer{} + s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if remaining := atomic.AddInt32(&s.failCount, -1); remaining >= 0 { + w.WriteHeader(http.StatusInternalServerError) + return + } + body, _ := io.ReadAll(r.Body) + s.mu.Lock() + s.payloads = append(s.payloads, body) + s.mu.Unlock() + w.WriteHeader(http.StatusOK) + })) + return s } -func (m *mockOTLPSender) getSendAttempts() int { - m.mu.Lock() - defer m.mu.Unlock() - return m.sendAttempts +func (s *testOTLPServer) getPayloads() [][]byte { + s.mu.Lock() + defer s.mu.Unlock() + cp := make([][]byte, len(s.payloads)) + copy(cp, s.payloads) + return cp } -func (m *mockOTLPSender) getCalls() [][]byte { - m.mu.Lock() - defer m.mu.Unlock() - return m.calls +func (s *testOTLPServer) requestCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.payloads) } -func newTestOTLPWriter(t *testing.T, sender *mockOTLPSender, opts ...StartOption) *otlpTraceWriter { +func newTestOTLPWriter(t *testing.T, srv *testOTLPServer, opts ...StartOption) *otlpTraceWriter { t.Helper() cfg, err := newTestConfig(append(opts, func(c *config) { - c.transport = &simpleTransport{} + c.ddTransport = &simpleTransport{} })...) require.NoError(t, err) return &otlpTraceWriter{ config: cfg, - sender: sender, + client: srv.Client(), + traceURL: srv.URL, + headers: map[string]string{"Content-Type": "application/x-protobuf"}, resource: buildResource(cfg.internalConfig), scope: nil, spans: make([]*otlptrace.Span, 0), @@ -77,8 +85,9 @@ func TestOTLPWriterImplementsTraceWriter(t *testing.T) { } func TestOTLPWriterAdd(t *testing.T) { - sender := &mockOTLPSender{} - w := newTestOTLPWriter(t, sender) + srv := newTestOTLPServer() + defer srv.Close() + w := newTestOTLPWriter(t, srv) spans := []*Span{ newSpan("op1", "svc", "res1", 1, 1, 0), @@ -92,8 +101,9 @@ func TestOTLPWriterAdd(t *testing.T) { } func TestOTLPWriterAddMultiple(t *testing.T) { - sender := &mockOTLPSender{} - w := newTestOTLPWriter(t, sender) + srv := newTestOTLPServer() + defer srv.Close() + w := newTestOTLPWriter(t, srv) w.add([]*Span{newSpan("op1", "svc", "res", 1, 1, 0)}) w.add([]*Span{newSpan("op2", "svc", "res", 2, 1, 0)}) @@ -105,18 +115,20 @@ func TestOTLPWriterAddMultiple(t *testing.T) { } func TestOTLPWriterFlushEmpty(t *testing.T) { - sender := &mockOTLPSender{} - w := newTestOTLPWriter(t, sender) + srv := newTestOTLPServer() + defer srv.Close() + w := newTestOTLPWriter(t, srv) w.flush() w.wg.Wait() - assert.Equal(t, 0, sender.getSendAttempts()) + assert.Equal(t, 0, srv.requestCount()) } func TestOTLPWriterFlush(t *testing.T) { - sender := &mockOTLPSender{} - w := newTestOTLPWriter(t, sender) + srv := newTestOTLPServer() + defer srv.Close() + w := newTestOTLPWriter(t, srv) w.add([]*Span{ newSpan("op1", "svc", "res", 1, 1, 0), @@ -125,11 +137,11 @@ func TestOTLPWriterFlush(t *testing.T) { w.flush() w.wg.Wait() - calls := sender.getCalls() - require.Equal(t, 1, len(calls)) + payloads := srv.getPayloads() + require.Equal(t, 1, len(payloads)) var tracesData otlptrace.TracesData - err := proto.Unmarshal(calls[0], &tracesData) + err := proto.Unmarshal(payloads[0], &tracesData) require.NoError(t, err) rs := tracesData.ResourceSpans @@ -139,8 +151,9 @@ func TestOTLPWriterFlush(t *testing.T) { } func TestOTLPWriterFlushClearsSpans(t *testing.T) { - sender := &mockOTLPSender{} - w := newTestOTLPWriter(t, sender) + srv := newTestOTLPServer() + defer srv.Close() + w := newTestOTLPWriter(t, srv) w.add([]*Span{newSpan("op1", "svc", "res", 1, 1, 0)}) w.flush() @@ -153,7 +166,7 @@ func TestOTLPWriterFlushClearsSpans(t *testing.T) { // Second flush should be a no-op w.flush() w.wg.Wait() - assert.Equal(t, 1, sender.getSendAttempts()) + assert.Equal(t, 1, srv.requestCount()) } func TestOTLPWriterFlushRetries(t *testing.T) { @@ -179,35 +192,51 @@ func TestOTLPWriterFlushRetries(t *testing.T) { for _, tc := range testcases { name := fmt.Sprintf("retries=%d/fails=%d", tc.configRetries, tc.failCount) t.Run(name, func(t *testing.T) { - sender := &mockOTLPSender{failCount: tc.failCount} - w := newTestOTLPWriter(t, sender, func(c *config) { + var totalRequests int32 + srv := newTestOTLPServer() + atomic.StoreInt32(&srv.failCount, int32(tc.failCount)) + defer srv.Close() + + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&totalRequests, 1) + srv.Server.Config.Handler.ServeHTTP(w, r) + }) + countingSrv := httptest.NewServer(mux) + defer countingSrv.Close() + + w := newTestOTLPWriter(t, srv, func(c *config) { c.sendRetries = tc.configRetries c.internalConfig.SetRetryInterval(time.Millisecond, internalconfig.OriginCode) }) + w.traceURL = countingSrv.URL + w.client = countingSrv.Client() w.add([]*Span{newSpan("op", "svc", "res", 1, 1, 0)}) w.flush() w.wg.Wait() - assert.Equal(t, tc.expAttempts, sender.getSendAttempts()) - assert.Equal(t, tc.tracesSent, len(sender.getCalls()) > 0) + assert.Equal(t, int32(tc.expAttempts), atomic.LoadInt32(&totalRequests)) + assert.Equal(t, tc.tracesSent, len(srv.getPayloads()) > 0) }) } } func TestOTLPWriterStop(t *testing.T) { - sender := &mockOTLPSender{} - w := newTestOTLPWriter(t, sender) + srv := newTestOTLPServer() + defer srv.Close() + w := newTestOTLPWriter(t, srv) w.add([]*Span{newSpan("op", "svc", "res", 1, 1, 0)}) w.stop() - assert.Equal(t, 1, len(sender.getCalls())) + assert.Equal(t, 1, len(srv.getPayloads())) } func TestOTLPWriterConcurrency(t *testing.T) { - sender := &mockOTLPSender{} - w := newTestOTLPWriter(t, sender) + srv := newTestOTLPServer() + defer srv.Close() + w := newTestOTLPWriter(t, srv) const numAdders = 20 const spansPerAdder = 50 @@ -250,7 +279,7 @@ func TestOTLPWriterConcurrency(t *testing.T) { // Verify all sent payloads are valid protobuf totalSpans := 0 - for _, data := range sender.getCalls() { + for _, data := range srv.getPayloads() { var td otlptrace.TracesData err := proto.Unmarshal(data, &td) require.NoError(t, err) @@ -262,18 +291,3 @@ func TestOTLPWriterConcurrency(t *testing.T) { } assert.Equal(t, numAdders*spansPerAdder, totalSpans) } - -func TestOTLPWriterNoopSender(t *testing.T) { - cfg, err := newTestConfig(func(c *config) { - c.transport = &simpleTransport{} - }) - require.NoError(t, err) - - w := newOTLPTraceWriter(cfg) - - // simpleTransport doesn't implement otlpSender, so noopOTLPSender should be used - w.add([]*Span{newSpan("op", "svc", "res", 1, 1, 0)}) - w.flush() - w.wg.Wait() - // No panic, no actual send — the noop sender returns an error that gets logged -} From d7749bb1bcbaa0e423b77fc4c13dec9efb8e159e Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Wed, 25 Mar 2026 14:07:29 -0400 Subject: [PATCH 30/58] introduce new otlpTransport, plus tests --- ddtrace/tracer/otlp_transport.go | 48 +++++++++++++ ddtrace/tracer/otlp_transport_test.go | 97 +++++++++++++++++++++++++++ ddtrace/tracer/otlp_writer.go | 59 +++++----------- ddtrace/tracer/otlp_writer_test.go | 17 ++--- 4 files changed, 168 insertions(+), 53 deletions(-) create mode 100644 ddtrace/tracer/otlp_transport.go create mode 100644 ddtrace/tracer/otlp_transport_test.go diff --git a/ddtrace/tracer/otlp_transport.go b/ddtrace/tracer/otlp_transport.go new file mode 100644 index 00000000000..fe5b4187794 --- /dev/null +++ b/ddtrace/tracer/otlp_transport.go @@ -0,0 +1,48 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016 Datadog, Inc. + +package tracer + +import ( + "bytes" + "fmt" + "net/http" +) + +// otlpTransport sends protobuf-encoded OTLP payloads over HTTP. +// It is the OTLP counterpart to httpTransport (which handles Datadog-protocol traffic). +type otlpTransport struct { + client *http.Client + endpoint string + headers map[string]string +} + +func newOTLPTransport(client *http.Client, endpoint string, headers map[string]string) *otlpTransport { + return &otlpTransport{ + client: client, + endpoint: endpoint, + headers: headers, + } +} + +// send posts a protobuf-encoded payload to the configured OTLP endpoint. +func (t *otlpTransport) send(data []byte) error { + req, err := http.NewRequest("POST", t.endpoint, bytes.NewReader(data)) + if err != nil { + return fmt.Errorf("cannot create http request: %s", err) + } + for header, value := range t.headers { + req.Header.Set(header, value) + } + resp, err := t.client.Do(req) + if err != nil { + return err + } + resp.Body.Close() + if code := resp.StatusCode; code >= 400 { + return fmt.Errorf("HTTP %d: %s", code, http.StatusText(code)) + } + return nil +} diff --git a/ddtrace/tracer/otlp_transport_test.go b/ddtrace/tracer/otlp_transport_test.go new file mode 100644 index 00000000000..7fc337ee3d6 --- /dev/null +++ b/ddtrace/tracer/otlp_transport_test.go @@ -0,0 +1,97 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016 Datadog, Inc. + +package tracer + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOTLPTransportSendSuccess(t *testing.T) { + var received []byte + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + received, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + tr := newOTLPTransport(srv.Client(), srv.URL, map[string]string{"Content-Type": "application/x-protobuf"}) + err := tr.send([]byte("hello")) + require.NoError(t, err) + assert.Equal(t, []byte("hello"), received) +} + +func TestOTLPTransportSendHeaders(t *testing.T) { + var gotHeaders http.Header + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotHeaders = r.Header.Clone() + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + headers := map[string]string{ + "Content-Type": "application/x-protobuf", + "Api-Key": "secret", + "X-Custom": "value", + } + tr := newOTLPTransport(srv.Client(), srv.URL, headers) + err := tr.send([]byte("data")) + require.NoError(t, err) + + assert.Equal(t, "application/x-protobuf", gotHeaders.Get("Content-Type")) + assert.Equal(t, "secret", gotHeaders.Get("Api-Key")) + assert.Equal(t, "value", gotHeaders.Get("X-Custom")) +} + +func TestOTLPTransportSendHTTPMethod(t *testing.T) { + var gotMethod string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotMethod = r.Method + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + tr := newOTLPTransport(srv.Client(), srv.URL, nil) + err := tr.send([]byte("data")) + require.NoError(t, err) + assert.Equal(t, "POST", gotMethod) +} + +func TestOTLPTransportSendErrorStatus(t *testing.T) { + tests := []struct { + code int + text string + }{ + {http.StatusBadRequest, "Bad Request"}, + {http.StatusUnauthorized, "Unauthorized"}, + {http.StatusInternalServerError, "Internal Server Error"}, + {http.StatusServiceUnavailable, "Service Unavailable"}, + } + for _, tt := range tests { + t.Run(tt.text, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tt.code) + })) + defer srv.Close() + + tr := newOTLPTransport(srv.Client(), srv.URL, nil) + err := tr.send([]byte("data")) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.text) + }) + } +} + +func TestOTLPTransportSendConnectionError(t *testing.T) { + tr := newOTLPTransport(http.DefaultClient, "http://127.0.0.1:0/nonexistent", nil) + err := tr.send([]byte("data")) + require.Error(t, err) +} diff --git a/ddtrace/tracer/otlp_writer.go b/ddtrace/tracer/otlp_writer.go index d1b3ea1bfbf..121e385f3b2 100644 --- a/ddtrace/tracer/otlp_writer.go +++ b/ddtrace/tracer/otlp_writer.go @@ -6,9 +6,6 @@ package tracer import ( - "bytes" - "fmt" - "net/http" "slices" "sync" "time" @@ -24,29 +21,25 @@ import ( var _ traceWriter = (*otlpTraceWriter)(nil) type otlpTraceWriter struct { - config *config - client *http.Client - traceURL string - headers map[string]string - mu locking.Mutex - resource *otlpresource.Resource - scope *otlpcommon.InstrumentationScope - spans []*otlptrace.Span // +checklocks:mu - climit chan struct{} - wg sync.WaitGroup + config *config + transport *otlpTransport + mu locking.Mutex + resource *otlpresource.Resource + scope *otlpcommon.InstrumentationScope + spans []*otlptrace.Span // +checklocks:mu + climit chan struct{} + wg sync.WaitGroup } func newOTLPTraceWriter(c *config) *otlpTraceWriter { return &otlpTraceWriter{ - config: c, - client: c.httpClient, - traceURL: c.internalConfig.OTLPTraceURL(), - headers: c.internalConfig.OTLPHeaders(), - resource: buildResource(c.internalConfig), - scope: &otlpcommon.InstrumentationScope{Name: "dd-trace-go"}, - spans: make([]*otlptrace.Span, 0), - climit: make(chan struct{}, concurrentConnectionLimit), - wg: sync.WaitGroup{}, + config: c, + transport: newOTLPTransport(c.httpClient, c.internalConfig.OTLPTraceURL(), c.internalConfig.OTLPHeaders()), + resource: buildResource(c.internalConfig), + scope: &otlpcommon.InstrumentationScope{Name: "dd-trace-go"}, + spans: make([]*otlptrace.Span, 0), + climit: make(chan struct{}, concurrentConnectionLimit), + wg: sync.WaitGroup{}, } } @@ -62,26 +55,6 @@ func (w *otlpTraceWriter) add(spanList []*Span) { } } -// send posts a protobuf-encoded OTLP payload to the configured collector endpoint. -func (w *otlpTraceWriter) send(data []byte) error { - req, err := http.NewRequest("POST", w.traceURL, bytes.NewReader(data)) - if err != nil { - return fmt.Errorf("cannot create http request: %s", err) - } - for header, value := range w.headers { - req.Header.Set(header, value) - } - resp, err := w.client.Do(req) - if err != nil { - return err - } - resp.Body.Close() - if code := resp.StatusCode; code >= 400 { - return fmt.Errorf("HTTP %d: %s", code, http.StatusText(code)) - } - return nil -} - func (w *otlpTraceWriter) flush() { w.mu.Lock() if len(w.spans) == 0 { @@ -125,7 +98,7 @@ func (w *otlpTraceWriter) flush() { var sendErr error for attempt := 0; attempt <= w.config.sendRetries; attempt++ { log.Debug("OTLP: attempt %d to send payload: %d bytes, %d spans", attempt+1, len(b), spanCount) - sendErr = w.send(b) + sendErr = w.transport.send(b) if sendErr == nil { log.Debug("OTLP: sent traces after %d attempts", attempt+1) return diff --git a/ddtrace/tracer/otlp_writer_test.go b/ddtrace/tracer/otlp_writer_test.go index c97f5415c3d..cb8b9de5f0d 100644 --- a/ddtrace/tracer/otlp_writer_test.go +++ b/ddtrace/tracer/otlp_writer_test.go @@ -69,14 +69,12 @@ func newTestOTLPWriter(t *testing.T, srv *testOTLPServer, opts ...StartOption) * })...) require.NoError(t, err) return &otlpTraceWriter{ - config: cfg, - client: srv.Client(), - traceURL: srv.URL, - headers: map[string]string{"Content-Type": "application/x-protobuf"}, - resource: buildResource(cfg.internalConfig), - scope: nil, - spans: make([]*otlptrace.Span, 0), - climit: make(chan struct{}, concurrentConnectionLimit), + config: cfg, + transport: newOTLPTransport(srv.Client(), srv.URL, map[string]string{"Content-Type": "application/x-protobuf"}), + resource: buildResource(cfg.internalConfig), + scope: nil, + spans: make([]*otlptrace.Span, 0), + climit: make(chan struct{}, concurrentConnectionLimit), } } @@ -209,8 +207,7 @@ func TestOTLPWriterFlushRetries(t *testing.T) { c.sendRetries = tc.configRetries c.internalConfig.SetRetryInterval(time.Millisecond, internalconfig.OriginCode) }) - w.traceURL = countingSrv.URL - w.client = countingSrv.Client() + w.transport = newOTLPTransport(countingSrv.Client(), countingSrv.URL, map[string]string{"Content-Type": "application/x-protobuf"}) w.add([]*Span{newSpan("op", "svc", "res", 1, 1, 0)}) w.flush() From dcd8a10dab8c3021a488c6c6ecfed1260b0ca712 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Wed, 25 Mar 2026 14:09:43 -0400 Subject: [PATCH 31/58] cleanup setting transport on tracer.config in non-otlp conditions --- ddtrace/tracer/option.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 1dd4d8c75c3..a2a972b557c 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -529,12 +529,11 @@ func apmTracingDisabled(c *config) { c.internalConfig.SetRuntimeMetricsV2Enabled(false, internalconfig.OriginCalculated) } -// resolveTraceTransport returns the trace URL and headers for the transport -// based on whether OTLP export mode is active. +// resolveTraceTransport returns the trace URL and headers for the Datadog +// agent transport. In OTLP export mode the ddTransport is not used for trace +// sending (otlpTransport handles that), but it may still be used for stats +// and agent discovery, so it always points at the DD agent. func resolveTraceTransport(cfg *internalconfig.Config) (traceURL string, headers map[string]string) { - if cfg.OTLPExportMode() { - return cfg.OTLPTraceURL(), cfg.OTLPHeaders() - } agentURL := cfg.AgentURL().String() traceURL = agentURL + tracesAPIPath if cfg.TraceProtocol() == traceProtocolV1 { From 6dbab086144b59faf86736a31abf8c48f27ba3da Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Wed, 25 Mar 2026 14:12:28 -0400 Subject: [PATCH 32/58] fix imports --- ddtrace/tracer/otlp_writer.go | 5 +++-- ddtrace/tracer/otlp_writer_test.go | 4 ++-- ddtrace/tracer/span_to_otlp.go | 4 ++-- ddtrace/tracer/span_to_otlp_test.go | 6 ++---- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ddtrace/tracer/otlp_writer.go b/ddtrace/tracer/otlp_writer.go index 121e385f3b2..8b98837f7cc 100644 --- a/ddtrace/tracer/otlp_writer.go +++ b/ddtrace/tracer/otlp_writer.go @@ -10,12 +10,13 @@ import ( "sync" "time" - "github.com/DataDog/dd-trace-go/v2/internal/locking" - "github.com/DataDog/dd-trace-go/v2/internal/log" otlpcommon "go.opentelemetry.io/proto/otlp/common/v1" otlpresource "go.opentelemetry.io/proto/otlp/resource/v1" otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" "google.golang.org/protobuf/proto" + + "github.com/DataDog/dd-trace-go/v2/internal/locking" + "github.com/DataDog/dd-trace-go/v2/internal/log" ) var _ traceWriter = (*otlpTraceWriter)(nil) diff --git a/ddtrace/tracer/otlp_writer_test.go b/ddtrace/tracer/otlp_writer_test.go index cb8b9de5f0d..a646434e1df 100644 --- a/ddtrace/tracer/otlp_writer_test.go +++ b/ddtrace/tracer/otlp_writer_test.go @@ -15,12 +15,12 @@ import ( "testing" "time" - internalconfig "github.com/DataDog/dd-trace-go/v2/internal/config" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" "google.golang.org/protobuf/proto" + + internalconfig "github.com/DataDog/dd-trace-go/v2/internal/config" ) // testOTLPServer is a test HTTP server that captures OTLP payloads. diff --git a/ddtrace/tracer/span_to_otlp.go b/ddtrace/tracer/span_to_otlp.go index 3c2eadcdb00..ba07876e594 100644 --- a/ddtrace/tracer/span_to_otlp.go +++ b/ddtrace/tracer/span_to_otlp.go @@ -8,13 +8,13 @@ package tracer import ( "encoding/binary" - internalconfig "github.com/DataDog/dd-trace-go/v2/internal/config" - "github.com/DataDog/dd-trace-go/v2/internal/version" otlpcommon "go.opentelemetry.io/proto/otlp/common/v1" otlpresource "go.opentelemetry.io/proto/otlp/resource/v1" otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" "github.com/DataDog/dd-trace-go/v2/ddtrace/ext" + internalconfig "github.com/DataDog/dd-trace-go/v2/internal/config" + "github.com/DataDog/dd-trace-go/v2/internal/version" ) // ----------------------------------------------------------------------------- diff --git a/ddtrace/tracer/span_to_otlp_test.go b/ddtrace/tracer/span_to_otlp_test.go index 6bc6ef3b0ce..3de074624f5 100644 --- a/ddtrace/tracer/span_to_otlp_test.go +++ b/ddtrace/tracer/span_to_otlp_test.go @@ -11,15 +11,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - otlpcommon "go.opentelemetry.io/proto/otlp/common/v1" otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" - internalconfig "github.com/DataDog/dd-trace-go/v2/internal/config" - "github.com/DataDog/dd-trace-go/v2/internal/version" - "github.com/DataDog/dd-trace-go/v2/ddtrace/ext" + internalconfig "github.com/DataDog/dd-trace-go/v2/internal/config" "github.com/DataDog/dd-trace-go/v2/internal/samplernames" + "github.com/DataDog/dd-trace-go/v2/internal/version" ) func TestBuildResource(t *testing.T) { From d0765c8c18f65afbb09f3f81f7e35e7255d44464 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Wed, 25 Mar 2026 14:30:11 -0400 Subject: [PATCH 33/58] fix lint errors --- ddtrace/tracer/otlp_writer.go | 4 ++-- ddtrace/tracer/otlp_writer_test.go | 12 ++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/ddtrace/tracer/otlp_writer.go b/ddtrace/tracer/otlp_writer.go index 8b98837f7cc..914ec0daa65 100644 --- a/ddtrace/tracer/otlp_writer.go +++ b/ddtrace/tracer/otlp_writer.go @@ -104,10 +104,10 @@ func (w *otlpTraceWriter) flush() { log.Debug("OTLP: sent traces after %d attempts", attempt+1) return } - log.Error("OTLP: failure sending traces (attempt %d of %d): %v", attempt+1, w.config.sendRetries+1, sendErr) + log.Error("OTLP: failure sending traces (attempt %d of %d): %v", attempt+1, w.config.sendRetries+1, sendErr.Error()) time.Sleep(w.config.internalConfig.RetryInterval()) } - log.Error("OTLP: lost %d spans: %v", spanCount, sendErr) + log.Error("OTLP: lost %d spans: %v", spanCount, sendErr.Error()) }() } diff --git a/ddtrace/tracer/otlp_writer_test.go b/ddtrace/tracer/otlp_writer_test.go index a646434e1df..e4da1219513 100644 --- a/ddtrace/tracer/otlp_writer_test.go +++ b/ddtrace/tracer/otlp_writer_test.go @@ -245,26 +245,22 @@ func TestOTLPWriterConcurrency(t *testing.T) { var spansAdded int32 for range numAdders { - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { <-start for range spansPerAdder { w.add([]*Span{newSpan("op", "svc", "res", randUint64(), randUint64(), 0)}) atomic.AddInt32(&spansAdded, 1) } - }() + }) } for range numFlushers { - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { <-start for range 10 { w.flush() } - }() + }) } close(start) From acc9688967b654dd5b345be4a7996fc2237eeaff Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Wed, 25 Mar 2026 14:40:12 -0400 Subject: [PATCH 34/58] Add comment reminder about benchmarking protobuf --- ddtrace/tracer/otlp_writer_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ddtrace/tracer/otlp_writer_test.go b/ddtrace/tracer/otlp_writer_test.go index e4da1219513..7c8d21d8080 100644 --- a/ddtrace/tracer/otlp_writer_test.go +++ b/ddtrace/tracer/otlp_writer_test.go @@ -284,3 +284,5 @@ func TestOTLPWriterConcurrency(t *testing.T) { } assert.Equal(t, numAdders*spansPerAdder, totalSpans) } + +// TODO: Add benchmark tests. From 490d6f1459702eb01915cfadd977676ed1a9010b Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Wed, 25 Mar 2026 16:06:53 -0400 Subject: [PATCH 35/58] implement flush condition if hit payloadMaxSize --- ddtrace/tracer/otlp_writer.go | 28 ++++++++++++++--- ddtrace/tracer/otlp_writer_test.go | 50 ++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/ddtrace/tracer/otlp_writer.go b/ddtrace/tracer/otlp_writer.go index 914ec0daa65..44d9cf528a6 100644 --- a/ddtrace/tracer/otlp_writer.go +++ b/ddtrace/tracer/otlp_writer.go @@ -28,6 +28,7 @@ type otlpTraceWriter struct { resource *otlpresource.Resource scope *otlpcommon.InstrumentationScope spans []*otlptrace.Span // +checklocks:mu + buffSize int // +checklocks:mu — estimated serialized size in bytes climit chan struct{} wg sync.WaitGroup } @@ -40,20 +41,40 @@ func newOTLPTraceWriter(c *config) *otlpTraceWriter { scope: &otlpcommon.InstrumentationScope{Name: "dd-trace-go"}, spans: make([]*otlptrace.Span, 0), climit: make(chan struct{}, concurrentConnectionLimit), - wg: sync.WaitGroup{}, } } +// reset swaps out the current span buffer and returns it, resetting the +// writer to an empty state ready for new spans. +func (w *otlpTraceWriter) reset() []*otlptrace.Span { + old := w.spans + w.spans = make([]*otlptrace.Span, 0) + w.buffSize = proto.Size(&otlptrace.TracesData{ + ResourceSpans: []*otlptrace.ResourceSpans{{ + Resource: w.resource, + ScopeSpans: []*otlptrace.ScopeSpans{{ + Scope: w.scope, + }}, + }}, + }) + return old +} + func (w *otlpTraceWriter) add(spanList []*Span) { defaultServiceName := w.config.internalConfig.ServiceName() w.mu.Lock() - defer w.mu.Unlock() w.spans = slices.Grow(w.spans, len(spanList)) for _, span := range spanList { if otlpSpan := convertSpan(span, defaultServiceName); otlpSpan != nil { w.spans = append(w.spans, otlpSpan) + w.buffSize += proto.Size(otlpSpan) } } + needsFlush := w.buffSize > payloadSizeLimit + w.mu.Unlock() + if needsFlush { + w.flush() + } } func (w *otlpTraceWriter) flush() { @@ -62,8 +83,7 @@ func (w *otlpTraceWriter) flush() { w.mu.Unlock() return } - readySpans := w.spans - w.spans = make([]*otlptrace.Span, 0) + readySpans := w.reset() w.mu.Unlock() w.climit <- struct{}{} diff --git a/ddtrace/tracer/otlp_writer_test.go b/ddtrace/tracer/otlp_writer_test.go index 7c8d21d8080..5f9389d24bf 100644 --- a/ddtrace/tracer/otlp_writer_test.go +++ b/ddtrace/tracer/otlp_writer_test.go @@ -10,6 +10,7 @@ import ( "io" "net/http" "net/http/httptest" + "strings" "sync" "sync/atomic" "testing" @@ -167,6 +168,55 @@ func TestOTLPWriterFlushClearsSpans(t *testing.T) { assert.Equal(t, 1, srv.requestCount()) } +func TestOTLPWriterFlushOnSize(t *testing.T) { + t.Run("single large span triggers flush", func(t *testing.T) { + srv := newTestOTLPServer() + defer srv.Close() + w := newTestOTLPWriter(t, srv) + + bigSpan := newSpan("op", "svc", "res", 1, 1, 0) + bigSpan.meta["big"] = strings.Repeat("X", payloadSizeLimit+1) + w.add([]*Span{bigSpan}) + w.wg.Wait() + + assert.GreaterOrEqual(t, srv.requestCount(), 1) + w.mu.Lock() + assert.Equal(t, 0, len(w.spans)) + w.mu.Unlock() + }) + + t.Run("many small spans accumulate past limit", func(t *testing.T) { + srv := newTestOTLPServer() + defer srv.Close() + w := newTestOTLPWriter(t, srv) + + // Each span has ~1KB of meta, so we need ~payloadSizeLimit/1024 spans. + spanSize := 1024 + numSpans := (payloadSizeLimit / spanSize) + 1 + for i := range numSpans { + s := newSpan("op", "svc", "res", uint64(i+1), 1, 0) + s.meta["data"] = strings.Repeat("X", spanSize) + w.add([]*Span{s}) + } + w.wg.Wait() + + assert.GreaterOrEqual(t, srv.requestCount(), 1) + }) + + t.Run("under limit does not trigger flush", func(t *testing.T) { + srv := newTestOTLPServer() + defer srv.Close() + w := newTestOTLPWriter(t, srv) + + w.add([]*Span{newSpan("op", "svc", "res", 1, 1, 0)}) + + assert.Equal(t, 0, srv.requestCount()) + w.mu.Lock() + assert.Equal(t, 1, len(w.spans)) + w.mu.Unlock() + }) +} + func TestOTLPWriterFlushRetries(t *testing.T) { testcases := []struct { configRetries int From c07bf3ce9f754f5e4a29b7237d7bbeb681c256fb Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Wed, 25 Mar 2026 16:45:16 -0400 Subject: [PATCH 36/58] Write benchmarks for otlp trace writer --- ddtrace/tracer/otlp_writer_bench_test.go | 207 +++++++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 ddtrace/tracer/otlp_writer_bench_test.go diff --git a/ddtrace/tracer/otlp_writer_bench_test.go b/ddtrace/tracer/otlp_writer_bench_test.go new file mode 100644 index 00000000000..f0217612b67 --- /dev/null +++ b/ddtrace/tracer/otlp_writer_bench_test.go @@ -0,0 +1,207 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016 Datadog, Inc. + +package tracer + +import ( + "fmt" + "net/http" + "net/http/httptest" + "sync" + "testing" + + "github.com/stretchr/testify/require" + otlpcommon "go.opentelemetry.io/proto/otlp/common/v1" + otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" + "google.golang.org/protobuf/proto" +) + +func newBenchOTLPWriter(b *testing.B) *otlpTraceWriter { + b.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + b.Cleanup(srv.Close) + cfg, err := newTestConfig() + require.NoError(b, err) + return &otlpTraceWriter{ + config: cfg, + transport: newOTLPTransport(srv.Client(), srv.URL, map[string]string{"Content-Type": "application/x-protobuf"}), + resource: buildResource(cfg.internalConfig), + spans: make([]*otlptrace.Span, 0), + climit: make(chan struct{}, concurrentConnectionLimit), + } +} + +func BenchmarkOTLPTraceWriterAdd(b *testing.B) { + traceSizes := []struct { + name string + numSpans int + }{ + {"1span", 1}, + {"5spans", 5}, + {"10spans", 10}, + {"50spans", 50}, + } + + for _, size := range traceSizes { + b.Run(size.name, func(b *testing.B) { + writer := newBenchOTLPWriter(b) + + trace := make([]*Span, size.numSpans) + for i := range size.numSpans { + trace[i] = newBasicSpan("benchmark-span") + } + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + writer.add(trace) + } + }) + } +} + +func BenchmarkOTLPTraceWriterFlush(b *testing.B) { + writer := newBenchOTLPWriter(b) + trace := []*Span{newBasicSpan("flush-test")} + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + writer.add(trace) + writer.flush() + writer.wg.Wait() + } +} + +func BenchmarkOTLPProtoMarshal(b *testing.B) { + spanCounts := []struct { + name string + n int + }{ + {"1span", 1}, + {"10spans", 10}, + {"100spans", 100}, + {"1000spans", 1000}, + } + + for _, sc := range spanCounts { + b.Run(sc.name, func(b *testing.B) { + spans := make([]*otlptrace.Span, sc.n) + for i := range sc.n { + s := newBasicSpan("bench-span") + s.meta["key"] = "value" + spans[i] = convertSpan(s, "bench-svc") + } + tracesData := buildTracesData(spans) + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + proto.Marshal(tracesData) + } + }) + } + + b.Run("rich_spans", func(b *testing.B) { + spans := make([]*otlptrace.Span, 100) + for i := range 100 { + s := newSpan("op", "svc", "res", uint64(i+1), 1, 0) + for j := range 20 { + s.meta[fmt.Sprintf("key-%d", j)] = fmt.Sprintf("value-%d", j) + } + for j := range 5 { + s.metrics[fmt.Sprintf("metric-%d", j)] = float64(j) * 1.5 + } + s.spanLinks = append(s.spanLinks, SpanLink{ + TraceID: uint64(i + 100), + TraceIDHigh: uint64(i + 200), + SpanID: uint64(i + 300), + Attributes: map[string]string{"link-key": "link-val"}, + }) + spans[i] = convertSpan(s, "bench-svc") + } + tracesData := buildTracesData(spans) + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + proto.Marshal(tracesData) + } + }) +} + +func BenchmarkOTLPProtoSize(b *testing.B) { + spanCounts := []struct { + name string + n int + }{ + {"1span", 1}, + {"10spans", 10}, + {"100spans", 100}, + {"1000spans", 1000}, + } + + for _, sc := range spanCounts { + b.Run(sc.name, func(b *testing.B) { + spans := make([]*otlptrace.Span, sc.n) + for i := range sc.n { + s := newBasicSpan("bench-span") + spans[i] = convertSpan(s, "bench-svc") + } + tracesData := buildTracesData(spans) + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + proto.Size(tracesData) + } + }) + } +} + +func buildTracesData(spans []*otlptrace.Span) *otlptrace.TracesData { + return &otlptrace.TracesData{ + ResourceSpans: []*otlptrace.ResourceSpans{{ + Resource: buildResource(nil), + ScopeSpans: []*otlptrace.ScopeSpans{{ + Scope: &otlpcommon.InstrumentationScope{Name: "dd-trace-go"}, + Spans: spans, + }}, + }}, + } +} + +func BenchmarkOTLPTraceWriterConcurrent(b *testing.B) { + concurrencyLevels := []int{1, 2, 4, 8} + + for _, concurrency := range concurrencyLevels { + b.Run(fmt.Sprintf("concurrency_%d", concurrency), func(b *testing.B) { + writer := newBenchOTLPWriter(b) + trace := []*Span{newBasicSpan("concurrent-test")} + + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + var wg sync.WaitGroup + + for range concurrency { + wg.Go(func() { + writer.add(trace) + }) + } + + wg.Wait() + } + }) + } +} From 90dccdd71a62cb7bf23edfa732d65c1ddbd6259c Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Thu, 26 Mar 2026 12:51:09 -0400 Subject: [PATCH 37/58] fix nits from CI --- ddtrace/tracer/otlp_writer.go | 1 + ddtrace/tracer/otlp_writer_bench_test.go | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ddtrace/tracer/otlp_writer.go b/ddtrace/tracer/otlp_writer.go index 44d9cf528a6..733293188ae 100644 --- a/ddtrace/tracer/otlp_writer.go +++ b/ddtrace/tracer/otlp_writer.go @@ -46,6 +46,7 @@ func newOTLPTraceWriter(c *config) *otlpTraceWriter { // reset swaps out the current span buffer and returns it, resetting the // writer to an empty state ready for new spans. +// +checklocks:w.mu func (w *otlpTraceWriter) reset() []*otlptrace.Span { old := w.spans w.spans = make([]*otlptrace.Span, 0) diff --git a/ddtrace/tracer/otlp_writer_bench_test.go b/ddtrace/tracer/otlp_writer_bench_test.go index f0217612b67..41d27cba5b5 100644 --- a/ddtrace/tracer/otlp_writer_bench_test.go +++ b/ddtrace/tracer/otlp_writer_bench_test.go @@ -120,10 +120,10 @@ func BenchmarkOTLPProtoMarshal(b *testing.B) { s.metrics[fmt.Sprintf("metric-%d", j)] = float64(j) * 1.5 } s.spanLinks = append(s.spanLinks, SpanLink{ - TraceID: uint64(i + 100), + TraceID: uint64(i + 100), TraceIDHigh: uint64(i + 200), - SpanID: uint64(i + 300), - Attributes: map[string]string{"link-key": "link-val"}, + SpanID: uint64(i + 300), + Attributes: map[string]string{"link-key": "link-val"}, }) spans[i] = convertSpan(s, "bench-svc") } From 99ef3c8285bfc892bb60ffc6007041c660661f3e Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Thu, 26 Mar 2026 13:55:04 -0400 Subject: [PATCH 38/58] Add last span atttributes and attribute limit, as well as tracestate field --- ddtrace/tracer/otlp_writer.go | 3 +- ddtrace/tracer/otlp_writer_test.go | 10 +- ddtrace/tracer/span_to_otlp.go | 100 ++++++++++++++++- ddtrace/tracer/span_to_otlp_test.go | 164 ++++++++++++++++++++++++++++ 4 files changed, 270 insertions(+), 7 deletions(-) diff --git a/ddtrace/tracer/otlp_writer.go b/ddtrace/tracer/otlp_writer.go index 733293188ae..f0be44d2fbc 100644 --- a/ddtrace/tracer/otlp_writer.go +++ b/ddtrace/tracer/otlp_writer.go @@ -17,6 +17,7 @@ import ( "github.com/DataDog/dd-trace-go/v2/internal/locking" "github.com/DataDog/dd-trace-go/v2/internal/log" + "github.com/DataDog/dd-trace-go/v2/internal/version" ) var _ traceWriter = (*otlpTraceWriter)(nil) @@ -38,7 +39,7 @@ func newOTLPTraceWriter(c *config) *otlpTraceWriter { config: c, transport: newOTLPTransport(c.httpClient, c.internalConfig.OTLPTraceURL(), c.internalConfig.OTLPHeaders()), resource: buildResource(c.internalConfig), - scope: &otlpcommon.InstrumentationScope{Name: "dd-trace-go"}, + scope: &otlpcommon.InstrumentationScope{Name: "dd-trace-go", Version: version.Tag}, spans: make([]*otlptrace.Span, 0), climit: make(chan struct{}, concurrentConnectionLimit), } diff --git a/ddtrace/tracer/otlp_writer_test.go b/ddtrace/tracer/otlp_writer_test.go index 5f9389d24bf..e433289a031 100644 --- a/ddtrace/tracer/otlp_writer_test.go +++ b/ddtrace/tracer/otlp_writer_test.go @@ -18,10 +18,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + otlpcommon "go.opentelemetry.io/proto/otlp/common/v1" otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" "google.golang.org/protobuf/proto" internalconfig "github.com/DataDog/dd-trace-go/v2/internal/config" + "github.com/DataDog/dd-trace-go/v2/internal/version" ) // testOTLPServer is a test HTTP server that captures OTLP payloads. @@ -73,7 +75,7 @@ func newTestOTLPWriter(t *testing.T, srv *testOTLPServer, opts ...StartOption) * config: cfg, transport: newOTLPTransport(srv.Client(), srv.URL, map[string]string{"Content-Type": "application/x-protobuf"}), resource: buildResource(cfg.internalConfig), - scope: nil, + scope: &otlpcommon.InstrumentationScope{Name: "dd-trace-go", Version: version.Tag}, spans: make([]*otlptrace.Span, 0), climit: make(chan struct{}, concurrentConnectionLimit), } @@ -146,6 +148,12 @@ func TestOTLPWriterFlush(t *testing.T) { rs := tracesData.ResourceSpans require.Equal(t, 1, len(rs)) require.Equal(t, 1, len(rs[0].ScopeSpans)) + + scope := rs[0].ScopeSpans[0].Scope + require.NotNil(t, scope) + assert.Equal(t, "dd-trace-go", scope.Name) + assert.Equal(t, version.Tag, scope.Version) + assert.Equal(t, 2, len(rs[0].ScopeSpans[0].Spans)) } diff --git a/ddtrace/tracer/span_to_otlp.go b/ddtrace/tracer/span_to_otlp.go index ba07876e594..d4e813ebe87 100644 --- a/ddtrace/tracer/span_to_otlp.go +++ b/ddtrace/tracer/span_to_otlp.go @@ -7,6 +7,8 @@ package tracer import ( "encoding/binary" + "encoding/json" + "fmt" otlpcommon "go.opentelemetry.io/proto/otlp/common/v1" otlpresource "go.opentelemetry.io/proto/otlp/resource/v1" @@ -17,6 +19,8 @@ import ( "github.com/DataDog/dd-trace-go/v2/internal/version" ) +const maxAttributesCount = 128 + // ----------------------------------------------------------------------------- // Resource construction // ----------------------------------------------------------------------------- @@ -63,6 +67,7 @@ func convertSpan(s *Span, defaultServiceName string) *otlptrace.Span { Events: convertEvents(s), Links: convertSpanLinks(s.spanLinks), Status: convertSpanStatus(s), + TraceState: convertTraceState(s.context), } } @@ -150,15 +155,41 @@ func getSpanKind(s *Span) string { return s.meta[ext.SpanKind] } // +checklocksignore — Post-finish: reads finished span fields during payload encoding. func convertSpanAttributes(s *Span, defaultServiceName string) []*otlpcommon.KeyValue { - attributes := convertMapToOTLPAttributesString(s.meta) + n := len(s.meta) + len(s.metrics) + len(s.metaStruct) + 2 + if s.service != defaultServiceName { + n++ + } + attrs := make([]*otlpcommon.KeyValue, 0, min(n, maxAttributesCount)) + + // add appends a key-value pair and returns true if there is still room. + add := func(key string, val *otlpcommon.AnyValue) bool { + if val != nil { + attrs = append(attrs, &otlpcommon.KeyValue{Key: key, Value: val}) + } + return len(attrs) < maxAttributesCount + } + + add("operation.name", otlpStringValue(s.name)) + add("span.type", otlpStringValue(s.spanType)) + for key, value := range s.meta { + if !add(key, otlpStringValue(value)) { + return attrs + } + } for key, value := range s.metrics { - attributes = append(attributes, otlpKeyValue(key, otlpDoubleValue(value))) + if !add(key, otlpDoubleValue(value)) { + return attrs + } + } + for key, value := range s.metaStruct { + if !add(key, anyToOTLPValue(value)) { + return attrs + } } if s.service != defaultServiceName { - attributes = append(attributes, otlpKeyValue("service.name", otlpStringValue(s.service))) + add("service.name", otlpStringValue(s.service)) } - // TODO: add meta_struct attributes - return attributes + return attrs } func convertMapToOTLPAttributesString(ddAttributes map[string]string) []*otlpcommon.KeyValue { @@ -188,6 +219,13 @@ func convertEventAttributes(ddAttributes map[string]*spanEventAttribute) []*otlp return out } +func convertTraceState(ctx *SpanContext) string { + if ctx.trace == nil { + return "" + } + return ctx.trace.propagatingTag(tracestateHeader) +} + // --- AnyValue helpers --- func otlpKeyValue(key string, value *otlpcommon.AnyValue) *otlpcommon.KeyValue { @@ -226,6 +264,58 @@ func otlpArrayValue(arr *spanEventArrayAttribute) *otlpcommon.AnyValue { return &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_ArrayValue{ArrayValue: &otlpcommon.ArrayValue{Values: values}}} } +// anyToOTLPValue converts an arbitrary Go value to an OTLP AnyValue. +// Handles primitives, maps, and slices recursively. For types that don't map +// directly to OTLP, falls back to JSON-encoding as a string. +func anyToOTLPValue(v any) *otlpcommon.AnyValue { + switch val := v.(type) { + case string: + return otlpStringValue(val) + case bool: + return otlpBoolValue(val) + case int64: + return otlpIntValue(val) + case float64: + return otlpDoubleValue(val) + case []any: + values := make([]*otlpcommon.AnyValue, 0, len(val)) + for _, elem := range val { + if av := anyToOTLPValue(elem); av != nil { + values = append(values, av) + } + } + return &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_ArrayValue{ + ArrayValue: &otlpcommon.ArrayValue{Values: values}, + }} + case map[string]any: + kvs := make([]*otlpcommon.KeyValue, 0, len(val)) + for k, elem := range val { + if av := anyToOTLPValue(elem); av != nil { + kvs = append(kvs, otlpKeyValue(k, av)) + } + } + return &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_KvlistValue{ + KvlistValue: &otlpcommon.KeyValueList{Values: kvs}, + }} + case map[string]string: + kvs := make([]*otlpcommon.KeyValue, 0, len(val)) + for k, elem := range val { + kvs = append(kvs, otlpKeyValue(k, otlpStringValue(elem))) + } + return &otlpcommon.AnyValue{Value: &otlpcommon.AnyValue_KvlistValue{ + KvlistValue: &otlpcommon.KeyValueList{Values: kvs}, + }} + default: + // Types without a natural OTLP mapping (e.g. custom structs) are + // JSON-encoded into a string as a best-effort fallback. + b, err := json.Marshal(val) + if err != nil { + return otlpStringValue(fmt.Sprintf("%v", val)) + } + return otlpStringValue(string(b)) + } +} + func spanEventArrayAttributeValueToAnyValue(v *spanEventArrayAttributeValue) *otlpcommon.AnyValue { if v == nil { return nil diff --git a/ddtrace/tracer/span_to_otlp_test.go b/ddtrace/tracer/span_to_otlp_test.go index 3de074624f5..bd579649201 100644 --- a/ddtrace/tracer/span_to_otlp_test.go +++ b/ddtrace/tracer/span_to_otlp_test.go @@ -7,6 +7,7 @@ package tracer import ( "encoding/binary" + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -200,6 +201,69 @@ func TestConvertSpanAttributes(t *testing.T) { assert.Equal(t, "test", m["env"]) assert.Equal(t, 10.0, m["count"]) assert.Equal(t, 0.5, m["rate"]) + assert.Equal(t, "op", m["operation.name"]) + assert.Contains(t, m, "span.type") +} + +func TestConvertSpanAttributesWithMetaStruct(t *testing.T) { + s := newBasicSpan("op") + s.meta = map[string]string{"tag": "val"} + s.metaStruct = map[string]any{ + "nested": map[string]any{"a": "b"}, + "simple": map[string]string{"x": "y"}, + } + + attrs := convertSpanAttributes(s, "") + + var nestedKV, simpleKV *otlpcommon.KeyValue + for _, kv := range attrs { + switch kv.Key { + case "nested": + nestedKV = kv + case "simple": + simpleKV = kv + } + } + + require.NotNil(t, nestedKV, "nested metaStruct key should be present") + kvlist := nestedKV.Value.GetKvlistValue() + require.NotNil(t, kvlist) + assert.Equal(t, "a", kvlist.Values[0].Key) + assert.Equal(t, "b", kvlist.Values[0].Value.GetStringValue()) + + require.NotNil(t, simpleKV, "simple metaStruct key should be present") + kvlist = simpleKV.Value.GetKvlistValue() + require.NotNil(t, kvlist) + assert.Equal(t, "x", kvlist.Values[0].Key) + assert.Equal(t, "y", kvlist.Values[0].Value.GetStringValue()) +} + +func TestConvertSpanAttributesMaxLimit(t *testing.T) { + s := newBasicSpan("op") + s.meta = make(map[string]string, 200) + for i := range 200 { + s.meta[fmt.Sprintf("key-%d", i)] = "val" + } + + attrs := convertSpanAttributes(s, "other-service") + assert.LessOrEqual(t, len(attrs), maxAttributesCount) +} + +func TestConvertSpanAttributesPriorityOrder(t *testing.T) { + s := newBasicSpan("op") + s.meta = make(map[string]string, maxAttributesCount) + for i := range maxAttributesCount { + s.meta[fmt.Sprintf("key-%d", i)] = "val" + } + s.metrics = map[string]float64{"should-be-dropped": 1.0} + + attrs := convertSpanAttributes(s, "") + m := keyValuesToMap(attrs) + + assert.Equal(t, "op", m["operation.name"], "operation.name should always be present") + assert.Contains(t, m, "span.type", "span.type should always be present") + assert.NotContains(t, m, "should-be-dropped", "metrics should be dropped when meta fills the limit") + assert.Equal(t, maxAttributesCount, len(attrs)) } func TestConvertSpanAttributesServiceNameOverride(t *testing.T) { @@ -328,6 +392,106 @@ func TestConvertSpanLinks_EmptyNil(t *testing.T) { assert.Empty(t, convertSpanLinks([]SpanLink{})) } +func TestConvertSpanTraceState(t *testing.T) { + t.Run("populated from span context", func(t *testing.T) { + s := newSpan("op", "svc", "res", 1, 1, 0) + setPropagatingTag(s.context, tracestateHeader, "dd=s:2;o:rum,othervendor=abc") + + otlpSpan := convertSpan(s, "svc") + require.NotNil(t, otlpSpan) + assert.Equal(t, "dd=s:2;o:rum,othervendor=abc", otlpSpan.TraceState) + }) + + t.Run("empty when no tracestate", func(t *testing.T) { + s := newSpan("op", "svc", "res", 1, 1, 0) + + otlpSpan := convertSpan(s, "svc") + require.NotNil(t, otlpSpan) + assert.Empty(t, otlpSpan.TraceState) + }) + + t.Run("empty when trace is nil", func(t *testing.T) { + s := newSpan("op", "svc", "res", 1, 1, 0) + s.context.trace = nil + + otlpSpan := convertSpan(s, "svc") + require.NotNil(t, otlpSpan) + assert.Empty(t, otlpSpan.TraceState) + }) +} + +func TestAnyToOTLPValue(t *testing.T) { + t.Run("string", func(t *testing.T) { + av := anyToOTLPValue("hello") + assert.Equal(t, "hello", av.GetStringValue()) + }) + + t.Run("bool", func(t *testing.T) { + av := anyToOTLPValue(true) + assert.Equal(t, true, av.GetBoolValue()) + }) + + t.Run("int64", func(t *testing.T) { + av := anyToOTLPValue(int64(42)) + assert.Equal(t, int64(42), av.GetIntValue()) + }) + + t.Run("float64", func(t *testing.T) { + av := anyToOTLPValue(3.14) + assert.Equal(t, 3.14, av.GetDoubleValue()) + }) + + t.Run("[]any", func(t *testing.T) { + av := anyToOTLPValue([]any{"a", int64(1), true}) + arr := av.GetArrayValue() + require.NotNil(t, arr) + require.Len(t, arr.Values, 3) + assert.Equal(t, "a", arr.Values[0].GetStringValue()) + assert.Equal(t, int64(1), arr.Values[1].GetIntValue()) + assert.Equal(t, true, arr.Values[2].GetBoolValue()) + }) + + t.Run("map[string]any", func(t *testing.T) { + av := anyToOTLPValue(map[string]any{"k": "v"}) + kvlist := av.GetKvlistValue() + require.NotNil(t, kvlist) + require.Len(t, kvlist.Values, 1) + assert.Equal(t, "k", kvlist.Values[0].Key) + assert.Equal(t, "v", kvlist.Values[0].Value.GetStringValue()) + }) + + t.Run("map[string]string", func(t *testing.T) { + av := anyToOTLPValue(map[string]string{"x": "y"}) + kvlist := av.GetKvlistValue() + require.NotNil(t, kvlist) + require.Len(t, kvlist.Values, 1) + assert.Equal(t, "x", kvlist.Values[0].Key) + assert.Equal(t, "y", kvlist.Values[0].Value.GetStringValue()) + }) + + t.Run("nested map", func(t *testing.T) { + av := anyToOTLPValue(map[string]any{ + "triggers": []any{map[string]any{"id": "1"}}, + }) + kvlist := av.GetKvlistValue() + require.NotNil(t, kvlist) + require.Len(t, kvlist.Values, 1) + assert.Equal(t, "triggers", kvlist.Values[0].Key) + inner := kvlist.Values[0].Value.GetArrayValue() + require.NotNil(t, inner) + require.Len(t, inner.Values, 1) + innerMap := inner.Values[0].GetKvlistValue() + require.NotNil(t, innerMap) + assert.Equal(t, "1", innerMap.Values[0].Value.GetStringValue()) + }) + + t.Run("default JSON fallback", func(t *testing.T) { + type custom struct{ A string } + av := anyToOTLPValue(custom{A: "test"}) + assert.Contains(t, av.GetStringValue(), `"A":"test"`) + }) +} + // keyValuesToMap converts []*otlpcommon.KeyValue into a map for easier assertion. // Values are returned as any (string or float64 for double). func keyValuesToMap(kvs []*otlpcommon.KeyValue) map[string]any { From ddfc0e2b079902edbbd11fced0166bb3382699e3 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Thu, 26 Mar 2026 14:41:29 -0400 Subject: [PATCH 39/58] fix checklocks on buffer --- ddtrace/tracer/otlp_writer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/tracer/otlp_writer.go b/ddtrace/tracer/otlp_writer.go index f0be44d2fbc..01869844f83 100644 --- a/ddtrace/tracer/otlp_writer.go +++ b/ddtrace/tracer/otlp_writer.go @@ -29,7 +29,7 @@ type otlpTraceWriter struct { resource *otlpresource.Resource scope *otlpcommon.InstrumentationScope spans []*otlptrace.Span // +checklocks:mu - buffSize int // +checklocks:mu — estimated serialized size in bytes + buffSize int // +checklocks:mu climit chan struct{} wg sync.WaitGroup } From c371138f9b5a54bb55ad37e3c39e82583beee339 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Thu, 26 Mar 2026 14:43:54 -0400 Subject: [PATCH 40/58] fix gofmt otlp_writer.go --- ddtrace/tracer/otlp_writer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/tracer/otlp_writer.go b/ddtrace/tracer/otlp_writer.go index 01869844f83..76c81cf6c78 100644 --- a/ddtrace/tracer/otlp_writer.go +++ b/ddtrace/tracer/otlp_writer.go @@ -29,7 +29,7 @@ type otlpTraceWriter struct { resource *otlpresource.Resource scope *otlpcommon.InstrumentationScope spans []*otlptrace.Span // +checklocks:mu - buffSize int // +checklocks:mu + buffSize int // +checklocks:mu climit chan struct{} wg sync.WaitGroup } From cd63872df74966cc0ef7cc45c05cbd7ae34d7b11 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Fri, 27 Mar 2026 14:07:00 -0400 Subject: [PATCH 41/58] Fix converSpanAttribute --- ddtrace/tracer/span_to_otlp.go | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/ddtrace/tracer/span_to_otlp.go b/ddtrace/tracer/span_to_otlp.go index d4e813ebe87..afdc2388010 100644 --- a/ddtrace/tracer/span_to_otlp.go +++ b/ddtrace/tracer/span_to_otlp.go @@ -153,6 +153,15 @@ func getSpanKind(s *Span) string { return s.meta[ext.SpanKind] } // Attribute conversion (DD → OTLP KeyValue / AnyValue) // ----------------------------------------------------------------------------- +// addAttribute appends a key-value pair to attrs and returns true if there is +// still room for more attributes. +func addAttribute(attrs *[]*otlpcommon.KeyValue, key string, val *otlpcommon.AnyValue) bool { + if val != nil { + *attrs = append(*attrs, &otlpcommon.KeyValue{Key: key, Value: val}) + } + return len(*attrs) < maxAttributesCount +} + // +checklocksignore — Post-finish: reads finished span fields during payload encoding. func convertSpanAttributes(s *Span, defaultServiceName string) []*otlpcommon.KeyValue { n := len(s.meta) + len(s.metrics) + len(s.metaStruct) + 2 @@ -161,33 +170,31 @@ func convertSpanAttributes(s *Span, defaultServiceName string) []*otlpcommon.Key } attrs := make([]*otlpcommon.KeyValue, 0, min(n, maxAttributesCount)) - // add appends a key-value pair and returns true if there is still room. - add := func(key string, val *otlpcommon.AnyValue) bool { - if val != nil { - attrs = append(attrs, &otlpcommon.KeyValue{Key: key, Value: val}) - } - return len(attrs) < maxAttributesCount + if !addAttribute(&attrs, "operation.name", otlpStringValue(s.name)) { + return attrs + } + if !addAttribute(&attrs, "span.type", otlpStringValue(s.spanType)) { + return attrs } - - add("operation.name", otlpStringValue(s.name)) - add("span.type", otlpStringValue(s.spanType)) for key, value := range s.meta { - if !add(key, otlpStringValue(value)) { + if !addAttribute(&attrs, key, otlpStringValue(value)) { return attrs } } for key, value := range s.metrics { - if !add(key, otlpDoubleValue(value)) { + if !addAttribute(&attrs, key, otlpDoubleValue(value)) { return attrs } } for key, value := range s.metaStruct { - if !add(key, anyToOTLPValue(value)) { + if !addAttribute(&attrs, key, anyToOTLPValue(value)) { return attrs } } if s.service != defaultServiceName { - add("service.name", otlpStringValue(s.service)) + if !addAttribute(&attrs, "service.name", otlpStringValue(s.service)) { + return attrs + } } return attrs } From 12d75b3a6f0918577315ffad6eec6c1da69f6e18 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler <46911781+mtoffl01@users.noreply.github.com> Date: Fri, 27 Mar 2026 14:29:56 -0400 Subject: [PATCH 42/58] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Dario Castañé --- ddtrace/tracer/span_to_otlp.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ddtrace/tracer/span_to_otlp.go b/ddtrace/tracer/span_to_otlp.go index afdc2388010..7bc9ac90e61 100644 --- a/ddtrace/tracer/span_to_otlp.go +++ b/ddtrace/tracer/span_to_otlp.go @@ -87,7 +87,7 @@ func convertSpanLinks(links []SpanLink) []*otlptrace.Span_Link { if len(links) == 0 { return nil } - otlpLinks := make([]*otlptrace.Span_Link, 0) + otlpLinks := make([]*otlptrace.Span_Link, 0, len(links)) for _, link := range links { otlpLinks = append(otlpLinks, &otlptrace.Span_Link{ TraceId: convertTraceID(link.TraceIDHigh, link.TraceID), @@ -105,7 +105,7 @@ func convertEvents(s *Span) []*otlptrace.Span_Event { if len(s.spanEvents) == 0 { return nil } - events := make([]*otlptrace.Span_Event, 0) + events := make([]*otlptrace.Span_Event, 0, len(s.spanEvents)) for _, event := range s.spanEvents { events = append(events, &otlptrace.Span_Event{ Name: event.Name, From af69c6b204e9ec66b103eb26732e0ab6048c3e49 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Fri, 27 Mar 2026 14:33:01 -0400 Subject: [PATCH 43/58] Make sure newBenchOTLPWriter has scope --- ddtrace/tracer/otlp_writer_bench_test.go | 3 +++ ddtrace/tracer/otlp_writer_test.go | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ddtrace/tracer/otlp_writer_bench_test.go b/ddtrace/tracer/otlp_writer_bench_test.go index 41d27cba5b5..8f5552061f1 100644 --- a/ddtrace/tracer/otlp_writer_bench_test.go +++ b/ddtrace/tracer/otlp_writer_bench_test.go @@ -16,6 +16,8 @@ import ( otlpcommon "go.opentelemetry.io/proto/otlp/common/v1" otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" "google.golang.org/protobuf/proto" + + "github.com/DataDog/dd-trace-go/v2/internal/version" ) func newBenchOTLPWriter(b *testing.B) *otlpTraceWriter { @@ -30,6 +32,7 @@ func newBenchOTLPWriter(b *testing.B) *otlpTraceWriter { config: cfg, transport: newOTLPTransport(srv.Client(), srv.URL, map[string]string{"Content-Type": "application/x-protobuf"}), resource: buildResource(cfg.internalConfig), + scope: &otlpcommon.InstrumentationScope{Name: "dd-trace-go", Version: version.Tag}, spans: make([]*otlptrace.Span, 0), climit: make(chan struct{}, concurrentConnectionLimit), } diff --git a/ddtrace/tracer/otlp_writer_test.go b/ddtrace/tracer/otlp_writer_test.go index e433289a031..f2339530c9e 100644 --- a/ddtrace/tracer/otlp_writer_test.go +++ b/ddtrace/tracer/otlp_writer_test.go @@ -342,5 +342,3 @@ func TestOTLPWriterConcurrency(t *testing.T) { } assert.Equal(t, numAdders*spansPerAdder, totalSpans) } - -// TODO: Add benchmark tests. From 35cf849f997845fb18f3704ab09a4b5240f11e91 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Mon, 30 Mar 2026 09:33:17 -0400 Subject: [PATCH 44/58] Address agent UDS url issue: have otlp writer create its own client --- ddtrace/tracer/otlp_writer.go | 3 ++- ddtrace/tracer/otlp_writer_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/ddtrace/tracer/otlp_writer.go b/ddtrace/tracer/otlp_writer.go index 76c81cf6c78..aa9a8bcdc24 100644 --- a/ddtrace/tracer/otlp_writer.go +++ b/ddtrace/tracer/otlp_writer.go @@ -15,6 +15,7 @@ import ( otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" "google.golang.org/protobuf/proto" + "github.com/DataDog/dd-trace-go/v2/internal" "github.com/DataDog/dd-trace-go/v2/internal/locking" "github.com/DataDog/dd-trace-go/v2/internal/log" "github.com/DataDog/dd-trace-go/v2/internal/version" @@ -37,7 +38,7 @@ type otlpTraceWriter struct { func newOTLPTraceWriter(c *config) *otlpTraceWriter { return &otlpTraceWriter{ config: c, - transport: newOTLPTransport(c.httpClient, c.internalConfig.OTLPTraceURL(), c.internalConfig.OTLPHeaders()), + transport: newOTLPTransport(internal.DefaultHTTPClient(c.httpClientTimeout, false), c.internalConfig.OTLPTraceURL(), c.internalConfig.OTLPHeaders()), resource: buildResource(c.internalConfig), scope: &otlpcommon.InstrumentationScope{Name: "dd-trace-go", Version: version.Tag}, spans: make([]*otlptrace.Span, 0), diff --git a/ddtrace/tracer/otlp_writer_test.go b/ddtrace/tracer/otlp_writer_test.go index f2339530c9e..e853404a424 100644 --- a/ddtrace/tracer/otlp_writer_test.go +++ b/ddtrace/tracer/otlp_writer_test.go @@ -22,6 +22,7 @@ import ( otlptrace "go.opentelemetry.io/proto/otlp/trace/v1" "google.golang.org/protobuf/proto" + "github.com/DataDog/dd-trace-go/v2/internal" internalconfig "github.com/DataDog/dd-trace-go/v2/internal/config" "github.com/DataDog/dd-trace-go/v2/internal/version" ) @@ -342,3 +343,29 @@ func TestOTLPWriterConcurrency(t *testing.T) { } assert.Equal(t, numAdders*spansPerAdder, totalSpans) } + +// TestOTLPWriterDoesNotReuseAgentHTTPClient verifies that the OTLP writer +// creates its own HTTP client rather than reusing c.httpClient, which may +// have a UDS dialer configured for the Datadog agent. +func TestOTLPWriterDoesNotReuseAgentHTTPClient(t *testing.T) { + srv := newTestOTLPServer() + defer srv.Close() + + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", srv.URL) + + cfg, err := newTestConfig(func(c *config) { + // Simulate a UDS-based agent: set c.httpClient to a UDS client + // pointing at a non-existent socket. If the OTLP writer reuses + // this client, its requests will fail. + c.httpClient = internal.UDSClient("/tmp/nonexistent-agent.sock", 5*time.Second) + c.ddTransport = &simpleTransport{} + }) + require.NoError(t, err) + + w := newOTLPTraceWriter(cfg) + + w.add([]*Span{newBasicSpan("uds-regression-test")}) + w.stop() + + assert.Equal(t, 1, srv.requestCount(), "OTLP writer should reach TCP server, not the UDS agent socket") +} From 6480f53ef8be86de0ac336ffa4dd653956659ce6 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Fri, 3 Apr 2026 15:22:00 -0400 Subject: [PATCH 45/58] PR comments: promote service.name and operation.name attributes; omit ParentId from proto when 0 --- ddtrace/tracer/span_to_otlp.go | 24 +++++++++++++++++------- ddtrace/tracer/span_to_otlp_test.go | 17 +++++++++++++++++ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/ddtrace/tracer/span_to_otlp.go b/ddtrace/tracer/span_to_otlp.go index 7bc9ac90e61..38b457a4a30 100644 --- a/ddtrace/tracer/span_to_otlp.go +++ b/ddtrace/tracer/span_to_otlp.go @@ -58,7 +58,7 @@ func convertSpan(s *Span, defaultServiceName string) *otlptrace.Span { return &otlptrace.Span{ TraceId: convertTraceID(s.context.traceID.Upper(), s.context.traceID.Lower()), SpanId: convertSpanID(s.spanID), - ParentSpanId: convertSpanID(s.parentID), + ParentSpanId: convertParentSpanID(s.parentID), Name: s.resource, Kind: convertSpanKind(getSpanKind(s)), StartTimeUnixNano: uint64(s.start), @@ -129,6 +129,13 @@ func convertSpanID(spanID uint64) []byte { return b } +func convertParentSpanID(parentID uint64) []byte { + if parentID == 0 { + return nil + } + return convertSpanID(parentID) +} + func convertSpanKind(spanKind string) otlptrace.Span_SpanKind { switch spanKind { case ext.SpanKindInternal: @@ -164,7 +171,7 @@ func addAttribute(attrs *[]*otlpcommon.KeyValue, key string, val *otlpcommon.Any // +checklocksignore — Post-finish: reads finished span fields during payload encoding. func convertSpanAttributes(s *Span, defaultServiceName string) []*otlpcommon.KeyValue { - n := len(s.meta) + len(s.metrics) + len(s.metaStruct) + 2 + n := len(s.meta) + len(s.metrics) + len(s.metaStruct) + 3 if s.service != defaultServiceName { n++ } @@ -173,9 +180,17 @@ func convertSpanAttributes(s *Span, defaultServiceName string) []*otlpcommon.Key if !addAttribute(&attrs, "operation.name", otlpStringValue(s.name)) { return attrs } + if !addAttribute(&attrs, "resource.name", otlpStringValue(s.resource)) { + return attrs + } if !addAttribute(&attrs, "span.type", otlpStringValue(s.spanType)) { return attrs } + if s.service != defaultServiceName { + if !addAttribute(&attrs, "service.name", otlpStringValue(s.service)) { + return attrs + } + } for key, value := range s.meta { if !addAttribute(&attrs, key, otlpStringValue(value)) { return attrs @@ -191,11 +206,6 @@ func convertSpanAttributes(s *Span, defaultServiceName string) []*otlpcommon.Key return attrs } } - if s.service != defaultServiceName { - if !addAttribute(&attrs, "service.name", otlpStringValue(s.service)) { - return attrs - } - } return attrs } diff --git a/ddtrace/tracer/span_to_otlp_test.go b/ddtrace/tracer/span_to_otlp_test.go index bd579649201..fe99b78844a 100644 --- a/ddtrace/tracer/span_to_otlp_test.go +++ b/ddtrace/tracer/span_to_otlp_test.go @@ -99,6 +99,21 @@ func TestConvertSpan(t *testing.T) { assert.Equal(t, 42.5, attrs["metric.key"]) } +func TestConvertSpanParentSpanId(t *testing.T) { + t.Run("set when parent_id is non-zero", func(t *testing.T) { + s := newSpan("op", "svc", "res", 100, 200, 50) + otlp := convertSpan(s, "svc") + require.NotNil(t, otlp.ParentSpanId) + assert.Equal(t, uint64(50), binary.BigEndian.Uint64(otlp.ParentSpanId)) + }) + + t.Run("omitted when parent_id is zero", func(t *testing.T) { + s := newSpan("op", "svc", "res", 100, 200, 0) + otlp := convertSpan(s, "svc") + assert.Nil(t, otlp.ParentSpanId, "ParentSpanId must be omitted for root spans") + }) +} + func TestConvertSpanFiltersUnsampled(t *testing.T) { tests := []struct { name string @@ -202,6 +217,7 @@ func TestConvertSpanAttributes(t *testing.T) { assert.Equal(t, 10.0, m["count"]) assert.Equal(t, 0.5, m["rate"]) assert.Equal(t, "op", m["operation.name"]) + assert.Contains(t, m, "resource.name") assert.Contains(t, m, "span.type") } @@ -261,6 +277,7 @@ func TestConvertSpanAttributesPriorityOrder(t *testing.T) { m := keyValuesToMap(attrs) assert.Equal(t, "op", m["operation.name"], "operation.name should always be present") + assert.Contains(t, m, "resource.name", "resource.name should always be present") assert.Contains(t, m, "span.type", "span.type should always be present") assert.NotContains(t, m, "should-be-dropped", "metrics should be dropped when meta fills the limit") assert.Equal(t, maxAttributesCount, len(attrs)) From e8aab3acc2fafd621eba1cf2c5340229d89ff0a1 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Tue, 7 Apr 2026 14:20:23 -0400 Subject: [PATCH 46/58] Add new benchmarks to BENCHMARK_TARGETS --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8a3db13971..d3aaa83eea9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ variables: # This base image is created here: https://gitlab.ddbuild.io/DataDog/apm-reliability/benchmarking-platform/-/jobs/1422209418 BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-go-96166511 INDEX_FILE: index.txt - BENCHMARK_TARGETS: "BenchmarkStartRequestSpan|BenchmarkHttpServeTrace|BenchmarkTracerAddSpans|BenchmarkStartSpan|BenchmarkSingleSpanRetention|BenchmarkOTelApiWithCustomTags|BenchmarkInjectW3C|BenchmarkExtractW3C|BenchmarkPartialFlushing|BenchmarkConfig|BenchmarkStartSpanConfig|BenchmarkGraphQL|BenchmarkSampleWAFContext|BenchmarkCaptureStackTrace|BenchmarkSetTagString|BenchmarkSetTagStringPtr|BenchmarkSetTagMetric|BenchmarkSetTagStringer|BenchmarkSerializeSpanLinksInMeta|BenchmarkLogs|BenchmarkParallelLogs|BenchmarkMetrics|BenchmarkParallelMetrics|BenchmarkPayloadVersions" + BENCHMARK_TARGETS: "BenchmarkStartRequestSpan|BenchmarkHttpServeTrace|BenchmarkTracerAddSpans|BenchmarkStartSpan|BenchmarkSingleSpanRetention|BenchmarkOTelApiWithCustomTags|BenchmarkInjectW3C|BenchmarkExtractW3C|BenchmarkPartialFlushing|BenchmarkConfig|BenchmarkStartSpanConfig|BenchmarkGraphQL|BenchmarkSampleWAFContext|BenchmarkCaptureStackTrace|BenchmarkSetTagString|BenchmarkSetTagStringPtr|BenchmarkSetTagMetric|BenchmarkSetTagStringer|BenchmarkSerializeSpanLinksInMeta|BenchmarkLogs|BenchmarkParallelLogs|BenchmarkMetrics|BenchmarkParallelMetrics|BenchmarkPayloadVersions|BenchmarkOTLPTraceWriterAdd|BenchmarkOTLPTraceWriterFlush|BenchmarkOTLPProtoMarshal|BenchmarkOTLPProtoSize|BenchmarkOTLPTraceWriterConcurrent" workflow: auto_cancel: From 037eec3154c62016cb7f7636ac7298237b5c0026 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Tue, 7 Apr 2026 15:37:50 -0400 Subject: [PATCH 47/58] Fix: Capture response body in otlpTransport.send, in order to reuse the same TCP connection instead of creating a new one for every request. Also, add tests. --- ddtrace/tracer/otlp_transport.go | 2 ++ ddtrace/tracer/otlp_transport_test.go | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/ddtrace/tracer/otlp_transport.go b/ddtrace/tracer/otlp_transport.go index fe5b4187794..68dc6362326 100644 --- a/ddtrace/tracer/otlp_transport.go +++ b/ddtrace/tracer/otlp_transport.go @@ -8,6 +8,7 @@ package tracer import ( "bytes" "fmt" + "io" "net/http" ) @@ -40,6 +41,7 @@ func (t *otlpTransport) send(data []byte) error { if err != nil { return err } + _, _ = io.Copy(io.Discard, resp.Body) resp.Body.Close() if code := resp.StatusCode; code >= 400 { return fmt.Errorf("HTTP %d: %s", code, http.StatusText(code)) diff --git a/ddtrace/tracer/otlp_transport_test.go b/ddtrace/tracer/otlp_transport_test.go index 7fc337ee3d6..b0a0063bbd8 100644 --- a/ddtrace/tracer/otlp_transport_test.go +++ b/ddtrace/tracer/otlp_transport_test.go @@ -7,8 +7,10 @@ package tracer import ( "io" + "net" "net/http" "net/http/httptest" + "sync/atomic" "testing" "github.com/stretchr/testify/assert" @@ -95,3 +97,23 @@ func TestOTLPTransportSendConnectionError(t *testing.T) { err := tr.send([]byte("data")) require.Error(t, err) } + +func TestOTLPTransportConnectionReuse(t *testing.T) { + var connCount int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("response body that must be drained")) + })) + defer srv.Close() + srv.Config.ConnState = func(_ net.Conn, state http.ConnState) { + if state == http.StateNew { + atomic.AddInt64(&connCount, 1) + } + } + + tr := newOTLPTransport(srv.Client(), srv.URL, nil) + for i := 0; i < 5; i++ { + require.NoError(t, tr.send([]byte("data"))) + } + assert.Equal(t, int64(1), atomic.LoadInt64(&connCount), + "expected a single connection to be reused across sends") +} From ca17d6a9ee1124931ecdd40480be428993404f80 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Tue, 7 Apr 2026 16:36:57 -0400 Subject: [PATCH 48/58] fix lint problem in otlp_transport_test --- ddtrace/tracer/otlp_transport_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/tracer/otlp_transport_test.go b/ddtrace/tracer/otlp_transport_test.go index b0a0063bbd8..89cf7be94cc 100644 --- a/ddtrace/tracer/otlp_transport_test.go +++ b/ddtrace/tracer/otlp_transport_test.go @@ -111,7 +111,7 @@ func TestOTLPTransportConnectionReuse(t *testing.T) { } tr := newOTLPTransport(srv.Client(), srv.URL, nil) - for i := 0; i < 5; i++ { + for range 5 { require.NoError(t, tr.send([]byte("data"))) } assert.Equal(t, int64(1), atomic.LoadInt64(&connCount), From 63bc9ea480fb4821d29586db472e9ad65d790d11 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Tue, 7 Apr 2026 16:51:44 -0400 Subject: [PATCH 49/58] Fix race condition in TestOTLPTransportConnectionReuse --- ddtrace/tracer/otlp_transport_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ddtrace/tracer/otlp_transport_test.go b/ddtrace/tracer/otlp_transport_test.go index 89cf7be94cc..f5b59e9b8a5 100644 --- a/ddtrace/tracer/otlp_transport_test.go +++ b/ddtrace/tracer/otlp_transport_test.go @@ -100,15 +100,16 @@ func TestOTLPTransportSendConnectionError(t *testing.T) { func TestOTLPTransportConnectionReuse(t *testing.T) { var connCount int64 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("response body that must be drained")) })) - defer srv.Close() srv.Config.ConnState = func(_ net.Conn, state http.ConnState) { if state == http.StateNew { atomic.AddInt64(&connCount, 1) } } + srv.Start() + defer srv.Close() tr := newOTLPTransport(srv.Client(), srv.URL, nil) for range 5 { From 91fac2a25c5a762d1a8fd227128b7aa99de10c33 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Thu, 9 Apr 2026 14:06:05 -0400 Subject: [PATCH 50/58] drain otlpTransport response body after reading status code --- ddtrace/tracer/otlp_transport.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ddtrace/tracer/otlp_transport.go b/ddtrace/tracer/otlp_transport.go index 68dc6362326..34b278544ba 100644 --- a/ddtrace/tracer/otlp_transport.go +++ b/ddtrace/tracer/otlp_transport.go @@ -41,8 +41,10 @@ func (t *otlpTransport) send(data []byte) error { if err != nil { return err } - _, _ = io.Copy(io.Discard, resp.Body) - resp.Body.Close() + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + }() if code := resp.StatusCode; code >= 400 { return fmt.Errorf("HTTP %d: %s", code, http.StatusText(code)) } From 7c1298869388d076bc439628da6683d8c14ffcbe Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Thu, 9 Apr 2026 14:17:43 -0400 Subject: [PATCH 51/58] Cache base TracesData size --- ddtrace/tracer/otlp_writer.go | 26 ++++++---- ddtrace/tracer/otlp_writer_test.go | 83 +++++++++++++++++++++++++++++- 2 files changed, 97 insertions(+), 12 deletions(-) diff --git a/ddtrace/tracer/otlp_writer.go b/ddtrace/tracer/otlp_writer.go index aa9a8bcdc24..5def9c2ef33 100644 --- a/ddtrace/tracer/otlp_writer.go +++ b/ddtrace/tracer/otlp_writer.go @@ -31,17 +31,30 @@ type otlpTraceWriter struct { scope *otlpcommon.InstrumentationScope spans []*otlptrace.Span // +checklocks:mu buffSize int // +checklocks:mu + baseSize int climit chan struct{} wg sync.WaitGroup } func newOTLPTraceWriter(c *config) *otlpTraceWriter { + resource := buildResource(c.internalConfig) + scope := &otlpcommon.InstrumentationScope{Name: "dd-trace-go", Version: version.Tag} + baseSize := proto.Size(&otlptrace.TracesData{ + ResourceSpans: []*otlptrace.ResourceSpans{{ + Resource: resource, + ScopeSpans: []*otlptrace.ScopeSpans{{ + Scope: scope, + }}, + }}, + }) return &otlpTraceWriter{ config: c, transport: newOTLPTransport(internal.DefaultHTTPClient(c.httpClientTimeout, false), c.internalConfig.OTLPTraceURL(), c.internalConfig.OTLPHeaders()), - resource: buildResource(c.internalConfig), - scope: &otlpcommon.InstrumentationScope{Name: "dd-trace-go", Version: version.Tag}, + resource: resource, + scope: scope, spans: make([]*otlptrace.Span, 0), + buffSize: baseSize, + baseSize: baseSize, climit: make(chan struct{}, concurrentConnectionLimit), } } @@ -52,14 +65,7 @@ func newOTLPTraceWriter(c *config) *otlpTraceWriter { func (w *otlpTraceWriter) reset() []*otlptrace.Span { old := w.spans w.spans = make([]*otlptrace.Span, 0) - w.buffSize = proto.Size(&otlptrace.TracesData{ - ResourceSpans: []*otlptrace.ResourceSpans{{ - Resource: w.resource, - ScopeSpans: []*otlptrace.ScopeSpans{{ - Scope: w.scope, - }}, - }}, - }) + w.buffSize = w.baseSize return old } diff --git a/ddtrace/tracer/otlp_writer_test.go b/ddtrace/tracer/otlp_writer_test.go index e853404a424..83a391a00d7 100644 --- a/ddtrace/tracer/otlp_writer_test.go +++ b/ddtrace/tracer/otlp_writer_test.go @@ -72,12 +72,24 @@ func newTestOTLPWriter(t *testing.T, srv *testOTLPServer, opts ...StartOption) * c.ddTransport = &simpleTransport{} })...) require.NoError(t, err) + resource := buildResource(cfg.internalConfig) + scope := &otlpcommon.InstrumentationScope{Name: "dd-trace-go", Version: version.Tag} + baseSize := proto.Size(&otlptrace.TracesData{ + ResourceSpans: []*otlptrace.ResourceSpans{{ + Resource: resource, + ScopeSpans: []*otlptrace.ScopeSpans{{ + Scope: scope, + }}, + }}, + }) return &otlpTraceWriter{ config: cfg, transport: newOTLPTransport(srv.Client(), srv.URL, map[string]string{"Content-Type": "application/x-protobuf"}), - resource: buildResource(cfg.internalConfig), - scope: &otlpcommon.InstrumentationScope{Name: "dd-trace-go", Version: version.Tag}, + resource: resource, + scope: scope, spans: make([]*otlptrace.Span, 0), + buffSize: baseSize, + baseSize: baseSize, climit: make(chan struct{}, concurrentConnectionLimit), } } @@ -344,6 +356,73 @@ func TestOTLPWriterConcurrency(t *testing.T) { assert.Equal(t, numAdders*spansPerAdder, totalSpans) } +func TestOTLPWriterBuffSizeTracking(t *testing.T) { + srv := newTestOTLPServer() + defer srv.Close() + w := newTestOTLPWriter(t, srv) + + t.Run("initial buffSize equals baseSize", func(t *testing.T) { + w.mu.Lock() + assert.Equal(t, w.baseSize, w.buffSize) + assert.Greater(t, w.baseSize, 0) + w.mu.Unlock() + }) + + t.Run("add increases buffSize", func(t *testing.T) { + w.mu.Lock() + before := w.buffSize + w.mu.Unlock() + + w.add([]*Span{newSpan("op", "svc", "res", 1, 1, 0)}) + + w.mu.Lock() + assert.Greater(t, w.buffSize, before) + w.mu.Unlock() + }) + + t.Run("flush resets buffSize to baseSize", func(t *testing.T) { + w.flush() + w.wg.Wait() + + w.mu.Lock() + assert.Equal(t, w.baseSize, w.buffSize) + w.mu.Unlock() + }) + + t.Run("buffSize approximates actual marshal size", func(t *testing.T) { + spans := []*Span{ + newSpan("op1", "svc", "res", 10, 10, 0), + newSpan("op2", "svc", "res", 20, 10, 10), + newSpan("op3", "svc", "res", 30, 10, 10), + } + w.add(spans) + + w.mu.Lock() + estimated := w.buffSize + spansCopy := make([]*otlptrace.Span, len(w.spans)) + copy(spansCopy, w.spans) + w.mu.Unlock() + + actual := proto.Size(&otlptrace.TracesData{ + ResourceSpans: []*otlptrace.ResourceSpans{{ + Resource: w.resource, + ScopeSpans: []*otlptrace.ScopeSpans{{ + Scope: w.scope, + Spans: spansCopy, + }}, + }}, + }) + // The estimate is baseSize + sum(proto.Size(span)), which slightly + // undercounts because it doesn't include the varint length prefix for + // each span in the repeated field. Verify it's close but not over. + assert.InDelta(t, actual, estimated, float64(actual)*0.05, + "estimated %d should be within 5%% of actual %d", estimated, actual) + + w.flush() + w.wg.Wait() + }) +} + // TestOTLPWriterDoesNotReuseAgentHTTPClient verifies that the OTLP writer // creates its own HTTP client rather than reusing c.httpClient, which may // have a UDS dialer configured for the Datadog agent. From f577ca277f8e3559638e44d3bd3a0623b15bb327 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Thu, 9 Apr 2026 14:30:04 -0400 Subject: [PATCH 52/58] Use Span_SPAN_KIND_UNSPECIFIED for unknown span types --- ddtrace/tracer/span_to_otlp.go | 2 +- ddtrace/tracer/span_to_otlp_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ddtrace/tracer/span_to_otlp.go b/ddtrace/tracer/span_to_otlp.go index 38b457a4a30..11675b99548 100644 --- a/ddtrace/tracer/span_to_otlp.go +++ b/ddtrace/tracer/span_to_otlp.go @@ -149,7 +149,7 @@ func convertSpanKind(spanKind string) otlptrace.Span_SpanKind { case ext.SpanKindConsumer: return otlptrace.Span_SPAN_KIND_CONSUMER default: - return otlptrace.Span_SPAN_KIND_INTERNAL + return otlptrace.Span_SPAN_KIND_UNSPECIFIED } } diff --git a/ddtrace/tracer/span_to_otlp_test.go b/ddtrace/tracer/span_to_otlp_test.go index fe99b78844a..177f6826b71 100644 --- a/ddtrace/tracer/span_to_otlp_test.go +++ b/ddtrace/tracer/span_to_otlp_test.go @@ -174,8 +174,8 @@ func TestConvertSpanKind(t *testing.T) { {ext.SpanKindClient, otlptrace.Span_SPAN_KIND_CLIENT}, {ext.SpanKindProducer, otlptrace.Span_SPAN_KIND_PRODUCER}, {ext.SpanKindConsumer, otlptrace.Span_SPAN_KIND_CONSUMER}, - {"", otlptrace.Span_SPAN_KIND_INTERNAL}, - {"unknown", otlptrace.Span_SPAN_KIND_INTERNAL}, + {"", otlptrace.Span_SPAN_KIND_UNSPECIFIED}, + {"unknown", otlptrace.Span_SPAN_KIND_UNSPECIFIED}, } for _, tt := range tests { t.Run(tt.dd, func(t *testing.T) { From ceea3ddfacc00dc909a16216e111933cf87e8218 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Thu, 9 Apr 2026 14:45:17 -0400 Subject: [PATCH 53/58] wrap error in otlpTransport.send --- ddtrace/tracer/otlp_transport.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/tracer/otlp_transport.go b/ddtrace/tracer/otlp_transport.go index 34b278544ba..7a2e17622a4 100644 --- a/ddtrace/tracer/otlp_transport.go +++ b/ddtrace/tracer/otlp_transport.go @@ -32,7 +32,7 @@ func newOTLPTransport(client *http.Client, endpoint string, headers map[string]s func (t *otlpTransport) send(data []byte) error { req, err := http.NewRequest("POST", t.endpoint, bytes.NewReader(data)) if err != nil { - return fmt.Errorf("cannot create http request: %s", err) + return fmt.Errorf("cannot create http request: %w", err) } for header, value := range t.headers { req.Header.Set(header, value) From b3a521d369b5c63bf945ab4d8bb5937a724aef84 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Thu, 9 Apr 2026 15:59:30 -0400 Subject: [PATCH 54/58] enforce Content-Type header at the otlpTransport.send level --- ddtrace/tracer/otlp_transport.go | 17 +++++++++-------- ddtrace/tracer/otlp_transport_test.go | 14 ++++++-------- ddtrace/tracer/otlp_writer_bench_test.go | 2 +- ddtrace/tracer/otlp_writer_test.go | 4 ++-- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/ddtrace/tracer/otlp_transport.go b/ddtrace/tracer/otlp_transport.go index 7a2e17622a4..b4a1acc9efe 100644 --- a/ddtrace/tracer/otlp_transport.go +++ b/ddtrace/tracer/otlp_transport.go @@ -15,16 +15,16 @@ import ( // otlpTransport sends protobuf-encoded OTLP payloads over HTTP. // It is the OTLP counterpart to httpTransport (which handles Datadog-protocol traffic). type otlpTransport struct { - client *http.Client - endpoint string - headers map[string]string + client *http.Client + endpoint string + customHeaders map[string]string } -func newOTLPTransport(client *http.Client, endpoint string, headers map[string]string) *otlpTransport { +func newOTLPTransport(client *http.Client, endpoint string, customHeaders map[string]string) *otlpTransport { return &otlpTransport{ - client: client, - endpoint: endpoint, - headers: headers, + client: client, + endpoint: endpoint, + customHeaders: customHeaders, } } @@ -34,7 +34,8 @@ func (t *otlpTransport) send(data []byte) error { if err != nil { return fmt.Errorf("cannot create http request: %w", err) } - for header, value := range t.headers { + req.Header.Set("Content-Type", "application/x-protobuf") + for header, value := range t.customHeaders { req.Header.Set(header, value) } resp, err := t.client.Do(req) diff --git a/ddtrace/tracer/otlp_transport_test.go b/ddtrace/tracer/otlp_transport_test.go index f5b59e9b8a5..555ca873734 100644 --- a/ddtrace/tracer/otlp_transport_test.go +++ b/ddtrace/tracer/otlp_transport_test.go @@ -25,7 +25,7 @@ func TestOTLPTransportSendSuccess(t *testing.T) { })) defer srv.Close() - tr := newOTLPTransport(srv.Client(), srv.URL, map[string]string{"Content-Type": "application/x-protobuf"}) + tr := newOTLPTransport(srv.Client(), srv.URL, nil) err := tr.send([]byte("hello")) require.NoError(t, err) assert.Equal(t, []byte("hello"), received) @@ -39,16 +39,14 @@ func TestOTLPTransportSendHeaders(t *testing.T) { })) defer srv.Close() - headers := map[string]string{ - "Content-Type": "application/x-protobuf", - "Api-Key": "secret", - "X-Custom": "value", - } - tr := newOTLPTransport(srv.Client(), srv.URL, headers) + tr := newOTLPTransport(srv.Client(), srv.URL, map[string]string{ + "Api-Key": "secret", + "X-Custom": "value", + }) err := tr.send([]byte("data")) require.NoError(t, err) - assert.Equal(t, "application/x-protobuf", gotHeaders.Get("Content-Type")) + assert.Equal(t, "application/x-protobuf", gotHeaders.Get("Content-Type"), "default Content-Type must be set") assert.Equal(t, "secret", gotHeaders.Get("Api-Key")) assert.Equal(t, "value", gotHeaders.Get("X-Custom")) } diff --git a/ddtrace/tracer/otlp_writer_bench_test.go b/ddtrace/tracer/otlp_writer_bench_test.go index 8f5552061f1..8d8a4004c88 100644 --- a/ddtrace/tracer/otlp_writer_bench_test.go +++ b/ddtrace/tracer/otlp_writer_bench_test.go @@ -30,7 +30,7 @@ func newBenchOTLPWriter(b *testing.B) *otlpTraceWriter { require.NoError(b, err) return &otlpTraceWriter{ config: cfg, - transport: newOTLPTransport(srv.Client(), srv.URL, map[string]string{"Content-Type": "application/x-protobuf"}), + transport: newOTLPTransport(srv.Client(), srv.URL, nil), resource: buildResource(cfg.internalConfig), scope: &otlpcommon.InstrumentationScope{Name: "dd-trace-go", Version: version.Tag}, spans: make([]*otlptrace.Span, 0), diff --git a/ddtrace/tracer/otlp_writer_test.go b/ddtrace/tracer/otlp_writer_test.go index 83a391a00d7..8baf459c19f 100644 --- a/ddtrace/tracer/otlp_writer_test.go +++ b/ddtrace/tracer/otlp_writer_test.go @@ -84,7 +84,7 @@ func newTestOTLPWriter(t *testing.T, srv *testOTLPServer, opts ...StartOption) * }) return &otlpTraceWriter{ config: cfg, - transport: newOTLPTransport(srv.Client(), srv.URL, map[string]string{"Content-Type": "application/x-protobuf"}), + transport: newOTLPTransport(srv.Client(), srv.URL, nil), resource: resource, scope: scope, spans: make([]*otlptrace.Span, 0), @@ -278,7 +278,7 @@ func TestOTLPWriterFlushRetries(t *testing.T) { c.sendRetries = tc.configRetries c.internalConfig.SetRetryInterval(time.Millisecond, internalconfig.OriginCode) }) - w.transport = newOTLPTransport(countingSrv.Client(), countingSrv.URL, map[string]string{"Content-Type": "application/x-protobuf"}) + w.transport = newOTLPTransport(countingSrv.Client(), countingSrv.URL, nil) w.add([]*Span{newSpan("op", "svc", "res", 1, 1, 0)}) w.flush() From 6ec640512ec4cde27068a9ec337a99633a145286 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Tue, 14 Apr 2026 13:19:05 -0400 Subject: [PATCH 55/58] Address codex feedback on span_to_otlp --- ddtrace/tracer/span_to_otlp.go | 6 +++++- ddtrace/tracer/span_to_otlp_test.go | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/ddtrace/tracer/span_to_otlp.go b/ddtrace/tracer/span_to_otlp.go index 11675b99548..7ee01a0aebc 100644 --- a/ddtrace/tracer/span_to_otlp.go +++ b/ddtrace/tracer/span_to_otlp.go @@ -269,7 +269,7 @@ func otlpIntValue(i int64) *otlpcommon.AnyValue { } func otlpArrayValue(arr *spanEventArrayAttribute) *otlpcommon.AnyValue { - if arr == nil || len(arr.Values) == 0 { + if arr == nil { return &otlpcommon.AnyValue{} } values := make([]*otlpcommon.AnyValue, 0, len(arr.Values)) @@ -292,6 +292,10 @@ func anyToOTLPValue(v any) *otlpcommon.AnyValue { return otlpBoolValue(val) case int64: return otlpIntValue(val) + case uint64: + return otlpIntValue(int64(val)) + case float32: + return otlpDoubleValue(float64(val)) case float64: return otlpDoubleValue(val) case []any: diff --git a/ddtrace/tracer/span_to_otlp_test.go b/ddtrace/tracer/span_to_otlp_test.go index 177f6826b71..8446de5176d 100644 --- a/ddtrace/tracer/span_to_otlp_test.go +++ b/ddtrace/tracer/span_to_otlp_test.go @@ -453,6 +453,16 @@ func TestAnyToOTLPValue(t *testing.T) { assert.Equal(t, int64(42), av.GetIntValue()) }) + t.Run("uint64", func(t *testing.T) { + av := anyToOTLPValue(uint64(64)) + assert.Equal(t, int64(64), av.GetIntValue()) + }) + + t.Run("float32", func(t *testing.T) { + av := anyToOTLPValue(float32(2.5)) + assert.InDelta(t, 2.5, av.GetDoubleValue(), 1e-6) + }) + t.Run("float64", func(t *testing.T) { av := anyToOTLPValue(3.14) assert.Equal(t, 3.14, av.GetDoubleValue()) From df091b3ca3349fd399a5f50d0c074d2a49129b7d Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Tue, 14 Apr 2026 13:30:19 -0400 Subject: [PATCH 56/58] Fix nil check on otlpArrayValue converter --- ddtrace/tracer/span_to_otlp.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ddtrace/tracer/span_to_otlp.go b/ddtrace/tracer/span_to_otlp.go index 7ee01a0aebc..e01369d7c5b 100644 --- a/ddtrace/tracer/span_to_otlp.go +++ b/ddtrace/tracer/span_to_otlp.go @@ -230,7 +230,9 @@ func convertEventAttributes(ddAttributes map[string]*spanEventAttribute) []*otlp case spanEventAttributeTypeInt: out = append(out, otlpKeyValue(key, otlpIntValue(value.IntValue))) case spanEventAttributeTypeArray: - out = append(out, otlpKeyValue(key, otlpArrayValue(value.ArrayValue))) + if kv := otlpKeyValue(key, otlpArrayValue(value.ArrayValue)); kv != nil { + out = append(out, kv) + } } } return out @@ -270,7 +272,7 @@ func otlpIntValue(i int64) *otlpcommon.AnyValue { func otlpArrayValue(arr *spanEventArrayAttribute) *otlpcommon.AnyValue { if arr == nil { - return &otlpcommon.AnyValue{} + return nil } values := make([]*otlpcommon.AnyValue, 0, len(arr.Values)) for _, v := range arr.Values { From 92de62c60fdd5d489953b0a416647af271e44059 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Tue, 14 Apr 2026 13:31:59 -0400 Subject: [PATCH 57/58] Document maxAttributesCount --- ddtrace/tracer/span_to_otlp.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ddtrace/tracer/span_to_otlp.go b/ddtrace/tracer/span_to_otlp.go index e01369d7c5b..c64417c57f6 100644 --- a/ddtrace/tracer/span_to_otlp.go +++ b/ddtrace/tracer/span_to_otlp.go @@ -19,6 +19,8 @@ import ( "github.com/DataDog/dd-trace-go/v2/internal/version" ) +// Derived from the default max attributes count for OTLP spans. +// See https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#attribute-limits const maxAttributesCount = 128 // ----------------------------------------------------------------------------- From 016cb71a6f63a11bb43b4fb7cd9ac3957fa6bc29 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Tue, 14 Apr 2026 13:55:08 -0400 Subject: [PATCH 58/58] Restore default traceWriter test --- ddtrace/tracer/tracer_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ddtrace/tracer/tracer_test.go b/ddtrace/tracer/tracer_test.go index 021c492d9b1..fe399e943ad 100644 --- a/ddtrace/tracer/tracer_test.go +++ b/ddtrace/tracer/tracer_test.go @@ -1459,6 +1459,17 @@ func TestTracerEdgeSampler(t *testing.T) { } func TestOTLPExportMode(t *testing.T) { + t.Run("default mode uses agentTraceWriter and prioritySampler", func(t *testing.T) { + assert := assert.New(t) + tracer, err := newUnstartedTracer() + assert.NoError(err) + defer tracer.Stop() + _, isAgentWriter := tracer.traceWriter.(*agentTraceWriter) + assert.True(isAgentWriter, "expected agentTraceWriter in default mode") + _, isPriority := tracer.defaultSampler.(*prioritySampler) + assert.True(isPriority, "expected prioritySampler in default mode") + }) + t.Run("uses otlpTraceWriter and otelParentBasedAlwaysOnSampler", func(t *testing.T) { assert := assert.New(t) tracer, err := newUnstartedTracer(func(c *config) { c.internalConfig.SetOTLPExportMode(true, internalconfig.OriginCode) })