diff --git a/CODEOWNERS b/CODEOWNERS index 80e0b73fc9..fbfd9872a2 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -68,8 +68,9 @@ # llm observability /llmobs @DataDog/ml-observability @DataDog/apm-idm-go @DataDog/apm-go /llmobs/export @DataDog/trajectory @DataDog/ml-observability @DataDog/apm-go +/otlp/export @DataDog/trajectory @DataDog/ml-observability @DataDog/apm-go /internal/llmobs @DataDog/ml-observability @DataDog/apm-idm-go @DataDog/apm-go -/internal/llmobs/exportutil @DataDog/trajectory @DataDog/ml-observability @DataDog/apm-go +/internal/exportutil @DataDog/trajectory @DataDog/ml-observability @DataDog/apm-go /contrib/mark3labs/mcp-go @DataDog/ml-observability @DataDog/apm-idm-go /contrib/modelcontextprotocol/go-sdk @DataDog/ml-observability @DataDog/apm-idm-go diff --git a/internal/README.md b/internal/README.md index 3c248fb4f2..724dcfdb3d 100644 --- a/internal/README.md +++ b/internal/README.md @@ -30,6 +30,10 @@ It also holds `supported_configurations*`, which maintains which environment var `go run ./scripts/configinverter/main.go add DD_MY_NEW_KEY` +### Export Utilities + +Contains small helpers for the offline export clients ([llmobs/export](../llmobs/export/) and [otlp/export](../otlp/export/)). Both clients use bounded, UTF-8-safe response-body snippets (`Snippet`) and per-request failure aggregation (`Aggregate`); `otlp/export` additionally uses the bounded-retry engine over a POST closure (`Retry`) with a transient-vs-permanent classifier (`Retriable`, or a caller-supplied one), while `llmobs/export` retries via the internal LLM Obs transport. See [exportutil](./exportutil/). + ### Locking Locking functionality that serves as a replacement for `sync.mutex` and similar locking mechanisms. It enables checking for deadlocks and should be used instead of `sync`. For more information, read the [README](./locking/README.md). diff --git a/internal/llmobs/exportutil/exportutil.go b/internal/exportutil/exportutil.go similarity index 95% rename from internal/llmobs/exportutil/exportutil.go rename to internal/exportutil/exportutil.go index aedf03283a..d02c9a51de 100644 --- a/internal/llmobs/exportutil/exportutil.go +++ b/internal/exportutil/exportutil.go @@ -4,8 +4,8 @@ // Copyright 2026 Datadog, Inc. // Package exportutil holds small helpers shared by the offline export clients -// (llmobs/export and otlp/export): bounded response-body snippets and -// per-request failure aggregation. +// (llmobs/export and otlp/export): bounded response-body snippets, +// per-request failure aggregation, and a bounded HTTP retry engine. package exportutil import ( diff --git a/internal/llmobs/exportutil/exportutil_test.go b/internal/exportutil/exportutil_test.go similarity index 100% rename from internal/llmobs/exportutil/exportutil_test.go rename to internal/exportutil/exportutil_test.go diff --git a/internal/exportutil/retry.go b/internal/exportutil/retry.go new file mode 100644 index 0000000000..b7599e41f9 --- /dev/null +++ b/internal/exportutil/retry.go @@ -0,0 +1,118 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package exportutil + +import ( + "context" + "math/rand/v2" + "net/http" + "time" +) + +const ( + initialBackoff = 100 * time.Millisecond + maxBackoff = time.Second +) + +// jitter applies equal-jitter to d (half fixed, half random in [0, d/2]) so that +// many exporters hitting the same transient failure do not retry in lockstep. +func jitter(d time.Duration) time.Duration { + if d <= 0 { + return d + } + return d/2 + time.Duration(rand.Int64N(int64(d/2)+1)) +} + +// Result reports the outcome of a bounded Retry over a single request's POST. +type Result struct { + StatusCode int + Attempts int + Body []byte + Retriable bool +} + +// Retriable classifies a failed attempt. A caller-cancelled/expired context is +// not transient; status 0 (network error), 408, 429 and 5xx are transient; +// other 4xx are permanent. +func Retriable(ctx context.Context, status int) bool { + if ctx.Err() != nil { + return false + } + switch { + case status == 0: + return true + case status == http.StatusRequestTimeout, status == http.StatusTooManyRequests: + return true + case status >= 500 && status <= 599: + return true + default: + return false + } +} + +// Attempt is the outcome of a single do() call inside Retry. +type Attempt struct { + // Status is the HTTP status (0 for a network-level error). + Status int + // Body is the (bounded) response body. + Body []byte + // RetryAfter is a server-requested minimum delay before the next attempt + // (e.g. parsed from a Retry-After header); 0 falls back to exponential + // backoff. It is only honored when the attempt is retriable. + RetryAfter time.Duration + // Err is nil on success. + Err error +} + +// RetryOptions configures Retry. +type RetryOptions struct { + // MaxAttempts bounds the total number of do() calls (>=1). + MaxAttempts uint + // Retriable classifies a failed status as transient. Defaults to Retriable + // when nil, letting callers with stricter rules (e.g. the OTLP/HTTP spec) + // override the classification. + Retriable func(ctx context.Context, status int) bool +} + +// Retry runs do up to opts.MaxAttempts times with exponential backoff, retrying +// only transient failures (per opts.Retriable). A retriable attempt that reports +// a RetryAfter waits at least that long instead of the backoff. It returns a +// structured Result plus the error from the final attempt (nil on success). +func Retry(ctx context.Context, opts RetryOptions, do func(context.Context) Attempt) (Result, error) { + retriable := opts.Retriable + if retriable == nil { + retriable = Retriable + } + var res Result + backoff := initialBackoff + for attempt := 1; ; attempt++ { + res.Attempts = attempt + a := do(ctx) + res.StatusCode = a.Status + res.Body = a.Body + if a.Err == nil { + res.Retriable = false + return res, nil + } + res.Retriable = retriable(ctx, a.Status) + if !res.Retriable || uint(attempt) >= opts.MaxAttempts { + return res, a.Err + } + wait := jitter(backoff) + if a.RetryAfter > 0 { + wait = a.RetryAfter + } + select { + case <-ctx.Done(): + res.Retriable = false + return res, ctx.Err() + case <-time.After(wait): + } + if backoff *= 2; backoff > maxBackoff { + backoff = maxBackoff + } + } +} diff --git a/internal/exportutil/retry_test.go b/internal/exportutil/retry_test.go new file mode 100644 index 0000000000..e5e6327bd2 --- /dev/null +++ b/internal/exportutil/retry_test.go @@ -0,0 +1,103 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package exportutil + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRetry_SucceedsFirstTry(t *testing.T) { + calls := 0 + res, err := Retry(context.Background(), RetryOptions{MaxAttempts: 3}, func(context.Context) Attempt { + calls++ + return Attempt{Status: 200, Body: []byte("ok")} + }) + require.NoError(t, err) + assert.Equal(t, 1, calls) + assert.Equal(t, 1, res.Attempts) + assert.False(t, res.Retriable) + assert.Equal(t, 200, res.StatusCode) +} + +func TestRetry_StopsAtMaxAttemptsWithDefaultClassifier(t *testing.T) { + calls := 0 + res, err := Retry(context.Background(), RetryOptions{MaxAttempts: 3}, func(context.Context) Attempt { + calls++ + return Attempt{Status: 503, Err: errors.New("unavailable")} // default Retriable retries 5xx + }) + require.Error(t, err) + assert.Equal(t, 3, calls) // exhausted every attempt + assert.Equal(t, 3, res.Attempts) + assert.True(t, res.Retriable) + assert.Equal(t, 503, res.StatusCode) +} + +func TestRetry_NonRetriableStopsImmediately(t *testing.T) { + calls := 0 + res, err := Retry(context.Background(), RetryOptions{MaxAttempts: 3}, func(context.Context) Attempt { + calls++ + return Attempt{Status: 400, Err: errors.New("bad request")} + }) + require.Error(t, err) + assert.Equal(t, 1, calls) // 400 is permanent under the default classifier + assert.False(t, res.Retriable) +} + +func TestRetry_HonorsRetryAfterOverBackoff(t *testing.T) { + calls := 0 + start := time.Now() + _, err := Retry(context.Background(), RetryOptions{MaxAttempts: 2}, func(context.Context) Attempt { + calls++ + if calls == 1 { + return Attempt{Status: 503, RetryAfter: 400 * time.Millisecond, Err: errors.New("throttled")} + } + return Attempt{Status: 200} + }) + require.NoError(t, err) + assert.Equal(t, 2, calls) + // The wait must honor RetryAfter (~400ms), not the ~100ms exponential backoff. + assert.GreaterOrEqual(t, time.Since(start), 350*time.Millisecond) +} + +func TestRetry_ContextCancelInterruptsWait(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + start := time.Now() + go func() { + time.Sleep(30 * time.Millisecond) + cancel() + }() + _, err := Retry(ctx, RetryOptions{MaxAttempts: 5}, func(context.Context) Attempt { + return Attempt{Status: 503, RetryAfter: 10 * time.Second, Err: errors.New("throttled")} + }) + require.ErrorIs(t, err, context.Canceled) + assert.Less(t, time.Since(start), 2*time.Second) // did not sleep the full 10s +} + +func TestRetry_CustomRetriablePredicate(t *testing.T) { + only418 := func(_ context.Context, status int) bool { return status == 418 } + + calls := 0 + _, err := Retry(context.Background(), RetryOptions{MaxAttempts: 3, Retriable: only418}, func(context.Context) Attempt { + calls++ + return Attempt{Status: 500, Err: errors.New("err")} // not retriable under this predicate + }) + require.Error(t, err) + assert.Equal(t, 1, calls) + + calls = 0 + _, err = Retry(context.Background(), RetryOptions{MaxAttempts: 3, Retriable: only418}, func(context.Context) Attempt { + calls++ + return Attempt{Status: 418, Err: errors.New("teapot")} + }) + require.Error(t, err) + assert.Equal(t, 3, calls) // 418 retries under this predicate +} diff --git a/llmobs/export/export.go b/llmobs/export/export.go index 72986a0ae1..f2ed120f15 100644 --- a/llmobs/export/export.go +++ b/llmobs/export/export.go @@ -14,7 +14,7 @@ import ( "slices" "strings" - "github.com/DataDog/dd-trace-go/v2/internal/llmobs/exportutil" + "github.com/DataDog/dd-trace-go/v2/internal/exportutil" "github.com/DataDog/dd-trace-go/v2/internal/llmobs/transport" ) diff --git a/otlp/export/bench_test.go b/otlp/export/bench_test.go new file mode 100644 index 0000000000..42372de3cd --- /dev/null +++ b/otlp/export/bench_test.go @@ -0,0 +1,51 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package export_test + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + + tracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" + + "github.com/DataDog/dd-trace-go/v2/otlp/export" +) + +// discardTransport accepts every request and retains nothing, so the benchmark +// measures marshal/POST-assembly cost without network or bookkeeping noise. +type discardTransport struct{} + +func (discardTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Body != nil { + _, _ = io.Copy(io.Discard, req.Body) + _ = req.Body.Close() + } + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader("")), Header: http.Header{}}, nil +} + +func BenchmarkExportTraces(b *testing.B) { + c, err := export.NewTraceClient(export.Config{ + Site: "datadoghq.com", APIKey: "k", + HTTPClient: &http.Client{Transport: discardTransport{}}, + }) + if err != nil { + b.Fatal(err) + } + reqs := make([]*tracepb.ExportTraceServiceRequest, 50) + for i := range reqs { + reqs[i] = sampleTrace() + } + ctx := context.Background() + b.ReportAllocs() + for b.Loop() { + if _, err := c.ExportTraces(ctx, reqs); err != nil { + b.Fatal(err) + } + } +} diff --git a/otlp/export/client.go b/otlp/export/client.go new file mode 100644 index 0000000000..4161d28944 --- /dev/null +++ b/otlp/export/client.go @@ -0,0 +1,58 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package export + +import ( + "context" + "fmt" + "reflect" + + "google.golang.org/protobuf/proto" + + "github.com/DataDog/dd-trace-go/v2/internal/exportutil" +) + +// partialSuccessFunc decodes a signal's OTLP 200 response body, reporting how +// many records the intake rejected (0 if none) and any accompanying message. It +// returns an error when the body is not a decodable Export*ServiceResponse, so a +// non-OTLP 200 (e.g. a proxy/login page) is surfaced as a failed export rather +// than silently counted as zero rejections. +type partialSuccessFunc func(body []byte) (rejected int64, message string, err error) + +// exportEach posts each request atomically (one request -> one POST -> one +// result row), preserving input order by index. A 200 response that reports OTLP +// partial success (rejected_* > 0), or whose body does not decode as the +// expected OTLP response, is surfaced as a per-request error via partial. It +// returns a non-nil error if any request failed; per-request detail is in the +// result. +func exportEach[T proto.Message](ctx context.Context, t *rawTransport, reqs []T, partial partialSuccessFunc) (*ExportResult, error) { + res := &ExportResult{} + for i, r := range reqs { + if isNilMessage(r) { + res.Requests = append(res.Requests, RequestResult{Index: i, Err: errNilRequest}) + continue + } + rr, body := t.export(ctx, r) + rr.Index = i + if rr.Err == nil { + switch rejected, msg, derr := partial(body); { + case derr != nil: + rr.Err = fmt.Errorf("otlp/export: response body is not a valid OTLP response: %w", derr) + case rejected > 0: + rr.Err = fmt.Errorf("otlp/export: intake reported partial success, %d record(s) rejected: %s", rejected, msg) + } + } + res.Requests = append(res.Requests, rr) + } + return res, exportutil.Aggregate(res.Failed(), len(res.Requests), "otlp/export") +} + +// isNilMessage reports whether v is a typed-nil proto message pointer, which +// would otherwise marshal to an empty body and be silently sent. +func isNilMessage[T proto.Message](v T) bool { + rv := reflect.ValueOf(v) + return rv.Kind() == reflect.Ptr && rv.IsNil() +} diff --git a/otlp/export/config.go b/otlp/export/config.go new file mode 100644 index 0000000000..03e2d56004 --- /dev/null +++ b/otlp/export/config.go @@ -0,0 +1,66 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package export + +import ( + "net/http" + "time" +) + +const ( + defaultSite = "datadoghq.com" + defaultMaxAttempts uint = 3 + + pathTraces = "/v1/traces" + pathMetrics = "/v1/metrics" + pathLogs = "/v1/logs" + + headerContentType = "Content-Type" + contentTypeProto = "application/x-protobuf" + headerAPIKey = "dd-api-key" + + // headerMetricConfig pins Datadog's OTLP metric intake to emit exponential + // histograms as distributions (DDSketch percentiles). Metrics + Datadog + // route only. + headerMetricConfig = "dd-otel-metric-config" + metricConfigDistributions = `{"histograms":{"mode":"distributions"}}` +) + +// Config configures an OTLP export client. A client targets exactly one +// destination and signal; build several clients for multi-destination export. +// +// Routing: +// - Datadog route (default): leave Endpoint empty and set Site + APIKey. The +// client derives https://otlp./v1/ and injects the dd-api-key +// header. +// - Collector/Agent route: set Endpoint to a base OTLP URL (the client +// appends /v1/). No Datadog auth is injected unless APIKey is also +// set (for a Datadog-compatible endpoint override). +type Config struct { + // Site is the Datadog site (e.g. "datadoghq.com"). Defaults to datadoghq.com. + // It is ignored when Endpoint is set. + Site string + // APIKey is the Datadog API key. Required for the Datadog route; when set it + // injects the dd-api-key header regardless of Endpoint. + APIKey string + // Endpoint is a base OTLP URL (scheme://host[:port]); the client appends the + // signal path. When empty, the endpoint is derived from Site. When set, it + // takes precedence over Site (the collector/Agent route). + Endpoint string + + // HTTPClient overrides the default HTTP client. + HTTPClient *http.Client + // Headers are extra request headers, applied last (they override defaults). + Headers map[string]string + // MaxAttempts bounds the total number of HTTP attempts per request, including + // the first (default 3, minimum 1). Set to 1 to disable retries. + MaxAttempts uint + // RequestTimeout bounds each individual HTTP attempt. When >0 it is applied to + // every attempt. When 0, a 10s default is applied only if the caller's context + // has no deadline of its own, so a caller passing a longer ctx deadline (for a + // large export or a slow collector) is not silently shortened. + RequestTimeout time.Duration +} diff --git a/otlp/export/doc.go b/otlp/export/doc.go new file mode 100644 index 0000000000..a2278e7736 --- /dev/null +++ b/otlp/export/doc.go @@ -0,0 +1,43 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +// Package export provides offline clients for exporting already-built OTLP +// traces, metrics, and logs to Datadog. +// +// It accepts OTLP collector proto requests +// (go.opentelemetry.io/proto/otlp/collector/{trace,metrics,logs}/v1) and posts +// them, protobuf-encoded, to Datadog's agentless OTLP intake (derived from the +// configured site as https://otlp./v1/{traces,metrics,logs}) or to a +// caller-provided collector/Agent endpoint. It uses a raw-proto transport and +// coexists with the OTel-SDK-based exporters in ddtrace/opentelemetry. +// +// It is an offline export API, not live instrumentation: the SDK owns endpoint +// derivation, auth/header injection, HTTP transport, retry classification, and +// structured results. Callers own proto construction, projection, temporality +// and histogram semantics, deterministic IDs, dedup, and durable retry. +// +// Each input request is treated atomically: one *Export*ServiceRequest becomes +// one POST and one result row. The SDK does not merge requests or split an +// oversized request. Caller-provided trace IDs, trace flags and tracestate are +// preserved as-is; the SDK does not infer W3C trace-ID randomness for +// reconstructed spans. +// +// Multiple destinations are modeled as one isolated client per destination. +// +// # Performance +// +// Requests are exported sequentially — one POST at a time — and each request +// holds its proto value plus its marshaled protobuf body in memory for the +// duration of that POST, so a slow destination serializes the rest of the batch. +// A Client is safe for concurrent use and holds no shared mutable state, so +// throughput for large exports is scaled by the caller: fan out across requests +// (or destinations) with a worker pool sized to taste, rather than relying on +// internal concurrency. See BenchmarkExportTraces for allocations/op and +// bytes/op. +// +// This package intentionally provides its own raw-proto transport rather than +// reusing ddtrace/tracer's OTLP writer; the owning team is recorded in +// CODEOWNERS so the two OTLP paths stay coherent. +package export diff --git a/otlp/export/export_test.go b/otlp/export/export_test.go new file mode 100644 index 0000000000..d82f4b3627 --- /dev/null +++ b/otlp/export/export_test.go @@ -0,0 +1,395 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package export_test + +import ( + "context" + "io" + "net/http" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" + "google.golang.org/protobuf/proto" + + logspb "go.opentelemetry.io/proto/otlp/collector/logs/v1" + metricspb "go.opentelemetry.io/proto/otlp/collector/metrics/v1" + tracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" + + "github.com/DataDog/dd-trace-go/v2/otlp/export" +) + +type fakeTransport struct { + mu sync.Mutex + requests []capturedRequest + responder func(attempt int) (int, string) +} + +type capturedRequest struct { + url string + headers http.Header + body []byte +} + +func (f *fakeTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if err := req.Context().Err(); err != nil { + return nil, err + } + f.mu.Lock() + attempt := len(f.requests) + var body []byte + if req.Body != nil { + body, _ = io.ReadAll(req.Body) + _ = req.Body.Close() + } + f.requests = append(f.requests, capturedRequest{url: req.URL.String(), headers: req.Header.Clone(), body: body}) + f.mu.Unlock() + + code, respBody := 200, "" + if f.responder != nil { + code, respBody = f.responder(attempt) + } + return &http.Response{ + StatusCode: code, + Body: io.NopCloser(strings.NewReader(respBody)), + Header: http.Header{"Content-Type": []string{"application/json"}}, + }, nil +} + +func (f *fakeTransport) captured() []capturedRequest { + f.mu.Lock() + defer f.mu.Unlock() + return f.requests +} + +func httpClient(f *fakeTransport) *http.Client { return &http.Client{Transport: f} } + +func sampleTrace() *tracepb.ExportTraceServiceRequest { + return &tracepb.ExportTraceServiceRequest{ + ResourceSpans: []*tracev1.ResourceSpans{{ + ScopeSpans: []*tracev1.ScopeSpans{{ + Spans: []*tracev1.Span{{ + TraceId: []byte("0123456789abcdef"), + SpanId: []byte("01234567"), + Name: "op", + }}, + }}, + }}, + } +} + +func TestExportTraces_DatadogRoute(t *testing.T) { + fake := &fakeTransport{} + c, err := export.NewTraceClient(export.Config{Site: "datadoghq.com", APIKey: "key", HTTPClient: httpClient(fake)}) + require.NoError(t, err) + + req := sampleTrace() + res, err := c.ExportTraces(context.Background(), []*tracepb.ExportTraceServiceRequest{req}) + require.NoError(t, err) + require.True(t, res.OK()) + require.Len(t, res.Requests, 1) + assert.Equal(t, 0, res.Requests[0].Index) + assert.Equal(t, 200, res.Requests[0].StatusCode) + assert.Equal(t, 1, res.Requests[0].Attempts) + + reqs := fake.captured() + require.Len(t, reqs, 1) + assert.Equal(t, "https://otlp.datadoghq.com/v1/traces", reqs[0].url) + assert.Equal(t, "key", reqs[0].headers.Get("dd-api-key")) + assert.Equal(t, "application/x-protobuf", reqs[0].headers.Get("Content-Type")) + assert.Empty(t, reqs[0].headers.Get("dd-otel-metric-config")) + + // Body round-trips (IDs preserved). + var got tracepb.ExportTraceServiceRequest + require.NoError(t, proto.Unmarshal(reqs[0].body, &got)) + assert.True(t, proto.Equal(req, &got)) +} + +func TestExportMetrics_AddsMetricConfigOnDatadogRoute(t *testing.T) { + fake := &fakeTransport{} + c, err := export.NewMetricClient(export.Config{Site: "us5.datadoghq.com", APIKey: "key", HTTPClient: httpClient(fake)}) + require.NoError(t, err) + + _, err = c.ExportMetrics(context.Background(), []*metricspb.ExportMetricsServiceRequest{{}}) + require.NoError(t, err) + + reqs := fake.captured() + require.Len(t, reqs, 1) + assert.Equal(t, "https://otlp.us5.datadoghq.com/v1/metrics", reqs[0].url) + assert.Equal(t, `{"histograms":{"mode":"distributions"}}`, reqs[0].headers.Get("dd-otel-metric-config")) + assert.Equal(t, "key", reqs[0].headers.Get("dd-api-key")) +} + +func TestExportMetrics_CollectorRouteNoAuthNoMetricConfig(t *testing.T) { + fake := &fakeTransport{} + c, err := export.NewMetricClient(export.Config{Endpoint: "http://collector:4318", HTTPClient: httpClient(fake)}) + require.NoError(t, err) + + _, err = c.ExportMetrics(context.Background(), []*metricspb.ExportMetricsServiceRequest{{}}) + require.NoError(t, err) + + reqs := fake.captured() + require.Len(t, reqs, 1) + assert.Equal(t, "http://collector:4318/v1/metrics", reqs[0].url) + assert.Empty(t, reqs[0].headers.Get("dd-api-key")) // no Datadog auth on collector route + assert.Empty(t, reqs[0].headers.Get("dd-otel-metric-config")) // metric config is Datadog-route only +} + +func TestExportMetrics_EndpointOverrideWithAPIKeyNoMetricConfig(t *testing.T) { + fake := &fakeTransport{} + // Datadog-compatible endpoint override + APIKey: auth is injected, but the + // dd-otel-metric-config header must not leak onto a non-derived endpoint. + c, err := export.NewMetricClient(export.Config{Endpoint: "http://collector:4318", APIKey: "key", HTTPClient: httpClient(fake)}) + require.NoError(t, err) + + _, err = c.ExportMetrics(context.Background(), []*metricspb.ExportMetricsServiceRequest{{}}) + require.NoError(t, err) + + reqs := fake.captured() + require.Len(t, reqs, 1) + assert.Equal(t, "key", reqs[0].headers.Get("dd-api-key")) + assert.Empty(t, reqs[0].headers.Get("dd-otel-metric-config")) +} + +func TestNew_RejectsSchemelessEndpoint(t *testing.T) { + _, err := export.NewTraceClient(export.Config{Endpoint: "collector:4318"}) + assert.Error(t, err) +} + +func TestNew_RejectsNonHTTPScheme(t *testing.T) { + _, err := export.NewTraceClient(export.Config{Endpoint: "grpc://collector:4317"}) + assert.Error(t, err) // OTLP/gRPC is not supported by the HTTP transport +} + +func TestExportTraces_PartialSuccessReportsError(t *testing.T) { + resp := &tracepb.ExportTraceServiceResponse{ + PartialSuccess: &tracepb.ExportTracePartialSuccess{RejectedSpans: 2, ErrorMessage: "2 spans dropped"}, + } + b, err := proto.Marshal(resp) + require.NoError(t, err) + fake := &fakeTransport{responder: func(int) (int, string) { return 200, string(b) }} + c, err := export.NewTraceClient(export.Config{Site: "datadoghq.com", APIKey: "key", HTTPClient: httpClient(fake)}) + require.NoError(t, err) + + res, err := c.ExportTraces(context.Background(), []*tracepb.ExportTraceServiceRequest{sampleTrace()}) + require.Error(t, err) // partial success surfaces as a failed request + require.Len(t, res.Requests, 1) + assert.Equal(t, 200, res.Requests[0].StatusCode) + require.Error(t, res.Requests[0].Err) + assert.Contains(t, res.Requests[0].Err.Error(), "partial success") +} + +func TestExportLogs_Endpoint(t *testing.T) { + fake := &fakeTransport{} + c, err := export.NewLogClient(export.Config{Site: "datadoghq.com", APIKey: "key", HTTPClient: httpClient(fake)}) + require.NoError(t, err) + + _, err = c.ExportLogs(context.Background(), []*logspb.ExportLogsServiceRequest{{}}) + require.NoError(t, err) + assert.Equal(t, "https://otlp.datadoghq.com/v1/logs", fake.captured()[0].url) +} + +func TestExportTraces_Non200IsFailure(t *testing.T) { + // A 202 (or any non-200 2xx) is not the OTLP success contract; report failure. + fake := &fakeTransport{responder: func(int) (int, string) { return 202, "" }} + c, err := export.NewTraceClient(export.Config{Site: "datadoghq.com", APIKey: "key", HTTPClient: httpClient(fake)}) + require.NoError(t, err) + + res, err := c.ExportTraces(context.Background(), []*tracepb.ExportTraceServiceRequest{sampleTrace()}) + require.Error(t, err) + require.Len(t, res.Requests, 1) + require.Error(t, res.Requests[0].Err) + assert.Equal(t, 202, res.Requests[0].StatusCode) +} + +func TestExportTraces_UndecodableBodyIsFailure(t *testing.T) { + // A 200 whose body is not a decodable OTLP response (e.g. a proxy/login page) + // must be a failed export, not silently counted as zero rejections. + fake := &fakeTransport{responder: func(int) (int, string) { return 200, "\x08\xff" }} // malformed protobuf + c, err := export.NewTraceClient(export.Config{Site: "datadoghq.com", APIKey: "key", HTTPClient: httpClient(fake)}) + require.NoError(t, err) + + res, err := c.ExportTraces(context.Background(), []*tracepb.ExportTraceServiceRequest{sampleTrace()}) + require.Error(t, err) + require.Len(t, res.Requests, 1) + require.Error(t, res.Requests[0].Err) + assert.Contains(t, res.Requests[0].Err.Error(), "not a valid OTLP response") +} + +func TestExportTraces_ForwardCompatibleResponseSucceeds(t *testing.T) { + // A 200 whose body carries a field unknown to ExportTraceServiceResponse (a + // forward-compatible extension) must still be treated as success: unknown + // protobuf fields are ignored, not used to reject the response. Bytes: + // tag(field 15, varint)=0x78, value=0x01. + fake := &fakeTransport{responder: func(int) (int, string) { return 200, "\x78\x01" }} + c, err := export.NewTraceClient(export.Config{Site: "datadoghq.com", APIKey: "key", HTTPClient: httpClient(fake)}) + require.NoError(t, err) + + res, err := c.ExportTraces(context.Background(), []*tracepb.ExportTraceServiceRequest{sampleTrace()}) + require.NoError(t, err) + require.Len(t, res.Requests, 1) + require.NoError(t, res.Requests[0].Err) +} + +func TestExportMetrics_PartialSuccessReportsError(t *testing.T) { + resp := &metricspb.ExportMetricsServiceResponse{ + PartialSuccess: &metricspb.ExportMetricsPartialSuccess{RejectedDataPoints: 4, ErrorMessage: "4 points dropped"}, + } + b, err := proto.Marshal(resp) + require.NoError(t, err) + fake := &fakeTransport{responder: func(int) (int, string) { return 200, string(b) }} + c, err := export.NewMetricClient(export.Config{Site: "datadoghq.com", APIKey: "key", HTTPClient: httpClient(fake)}) + require.NoError(t, err) + + res, err := c.ExportMetrics(context.Background(), []*metricspb.ExportMetricsServiceRequest{{}}) + require.Error(t, err) // rejected data points surface as a failed request + require.Len(t, res.Requests, 1) + require.Error(t, res.Requests[0].Err) + assert.Contains(t, res.Requests[0].Err.Error(), "partial success") +} + +func TestExportLogs_PartialSuccessReportsError(t *testing.T) { + resp := &logspb.ExportLogsServiceResponse{ + PartialSuccess: &logspb.ExportLogsPartialSuccess{RejectedLogRecords: 3, ErrorMessage: "3 logs dropped"}, + } + b, err := proto.Marshal(resp) + require.NoError(t, err) + fake := &fakeTransport{responder: func(int) (int, string) { return 200, string(b) }} + c, err := export.NewLogClient(export.Config{Site: "datadoghq.com", APIKey: "key", HTTPClient: httpClient(fake)}) + require.NoError(t, err) + + res, err := c.ExportLogs(context.Background(), []*logspb.ExportLogsServiceRequest{{}}) + require.Error(t, err) // rejected log records surface as a failed request + require.Len(t, res.Requests, 1) + require.Error(t, res.Requests[0].Err) + assert.Contains(t, res.Requests[0].Err.Error(), "partial success") +} + +func TestExportTraces_PerRequestRows(t *testing.T) { + fake := &fakeTransport{} + c, err := export.NewTraceClient(export.Config{Site: "datadoghq.com", APIKey: "key", HTTPClient: httpClient(fake)}) + require.NoError(t, err) + + res, err := c.ExportTraces(context.Background(), []*tracepb.ExportTraceServiceRequest{sampleTrace(), sampleTrace(), sampleTrace()}) + require.NoError(t, err) + require.Len(t, res.Requests, 3) // one row per request, not flattened spans + assert.Len(t, fake.captured(), 3) + for i, rr := range res.Requests { + assert.Equal(t, i, rr.Index) + } +} + +func TestExportTraces_RetryTransient(t *testing.T) { + fake := &fakeTransport{responder: func(int) (int, string) { return 503, "unavailable" }} + c, err := export.NewTraceClient(export.Config{Site: "datadoghq.com", APIKey: "key", HTTPClient: httpClient(fake), MaxAttempts: 3}) + require.NoError(t, err) + + res, err := c.ExportTraces(context.Background(), []*tracepb.ExportTraceServiceRequest{sampleTrace()}) + require.Error(t, err) + require.Len(t, res.Requests, 1) + assert.Equal(t, 3, res.Requests[0].Attempts) // total attempts == MaxAttempts + assert.True(t, res.Requests[0].Retriable) + assert.Equal(t, 503, res.Requests[0].StatusCode) +} + +func TestExportTraces_PermanentError(t *testing.T) { + fake := &fakeTransport{responder: func(int) (int, string) { return 400, "bad" }} + c, err := export.NewTraceClient(export.Config{Site: "datadoghq.com", APIKey: "key", HTTPClient: httpClient(fake), MaxAttempts: 3}) + require.NoError(t, err) + + res, err := c.ExportTraces(context.Background(), []*tracepb.ExportTraceServiceRequest{sampleTrace()}) + require.Error(t, err) + assert.Equal(t, 1, res.Requests[0].Attempts) // not retried + assert.False(t, res.Requests[0].Retriable) + assert.Equal(t, 400, res.Requests[0].StatusCode) +} + +func TestExportTraces_NonRetryableServerErrorNotRetried(t *testing.T) { + // 500 is a 5xx but not in the OTLP retryable set (429/502/503/504); it must + // not burn every attempt like the generic classifier would. + fake := &fakeTransport{responder: func(int) (int, string) { return 500, "boom" }} + c, err := export.NewTraceClient(export.Config{Site: "datadoghq.com", APIKey: "key", HTTPClient: httpClient(fake), MaxAttempts: 3}) + require.NoError(t, err) + + res, err := c.ExportTraces(context.Background(), []*tracepb.ExportTraceServiceRequest{sampleTrace()}) + require.Error(t, err) + assert.Equal(t, 1, res.Requests[0].Attempts) // 500 is permanent under OTLP rules + assert.False(t, res.Requests[0].Retriable) + assert.Equal(t, 500, res.Requests[0].StatusCode) +} + +func TestExportTraces_RetriesBadGateway(t *testing.T) { + // 502 is in the OTLP retryable set. + fake := &fakeTransport{responder: func(int) (int, string) { return 502, "" }} + c, err := export.NewTraceClient(export.Config{Site: "datadoghq.com", APIKey: "key", HTTPClient: httpClient(fake), MaxAttempts: 2}) + require.NoError(t, err) + + res, err := c.ExportTraces(context.Background(), []*tracepb.ExportTraceServiceRequest{sampleTrace()}) + require.Error(t, err) + assert.Equal(t, 2, res.Requests[0].Attempts) + assert.True(t, res.Requests[0].Retriable) +} + +func TestExportTraces_SurfacesDecodedStatusMessage(t *testing.T) { + // OTLP/HTTP error bodies are a google.rpc.Status protobuf; the snippet should + // show its message, not raw protobuf control bytes. + var status []byte + status = protowire.AppendTag(status, 1, protowire.VarintType) + status = protowire.AppendVarint(status, 3) + status = protowire.AppendTag(status, 2, protowire.BytesType) + status = protowire.AppendBytes(status, []byte("resource_spans[0] rejected: bad trace_id")) + + fake := &fakeTransport{responder: func(int) (int, string) { return 400, string(status) }} + c, err := export.NewTraceClient(export.Config{Site: "datadoghq.com", APIKey: "key", HTTPClient: httpClient(fake)}) + require.NoError(t, err) + + res, err := c.ExportTraces(context.Background(), []*tracepb.ExportTraceServiceRequest{sampleTrace()}) + require.Error(t, err) + assert.Equal(t, "resource_spans[0] rejected: bad trace_id", res.Requests[0].ResponseSnippet) +} + +func TestExportTraces_NilRequest(t *testing.T) { + fake := &fakeTransport{} + c, err := export.NewTraceClient(export.Config{Site: "datadoghq.com", APIKey: "key", HTTPClient: httpClient(fake)}) + require.NoError(t, err) + + res, err := c.ExportTraces(context.Background(), []*tracepb.ExportTraceServiceRequest{nil}) + require.Error(t, err) + require.Len(t, res.Requests, 1) + assert.Error(t, res.Requests[0].Err) + assert.Empty(t, fake.captured()) // nil request never sent +} + +func TestExportTraces_ContextCancelNotRetriable(t *testing.T) { + fake := &fakeTransport{} + c, err := export.NewTraceClient(export.Config{Site: "datadoghq.com", APIKey: "key", HTTPClient: httpClient(fake)}) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + res, err := c.ExportTraces(ctx, []*tracepb.ExportTraceServiceRequest{sampleTrace()}) + require.Error(t, err) + require.Len(t, res.Requests, 1) + assert.False(t, res.Requests[0].Retriable) +} + +func TestNew_RequiresAPIKeyOrEndpoint(t *testing.T) { + _, err := export.NewTraceClient(export.Config{Site: "datadoghq.com"}) + assert.Error(t, err) +} + +func TestNew_EndpointTrimsTrailingSlash(t *testing.T) { + fake := &fakeTransport{} + c, err := export.NewTraceClient(export.Config{Endpoint: "http://collector:4318/", HTTPClient: httpClient(fake)}) + require.NoError(t, err) + _, err = c.ExportTraces(context.Background(), []*tracepb.ExportTraceServiceRequest{sampleTrace()}) + require.NoError(t, err) + assert.Equal(t, "http://collector:4318/v1/traces", fake.captured()[0].url) +} diff --git a/otlp/export/log.go b/otlp/export/log.go new file mode 100644 index 0000000000..a85da0343d --- /dev/null +++ b/otlp/export/log.go @@ -0,0 +1,43 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package export + +import ( + "context" + + "google.golang.org/protobuf/proto" + + logspb "go.opentelemetry.io/proto/otlp/collector/logs/v1" +) + +// LogClient exports offline OTLP log requests. +type LogClient struct { + t *rawTransport +} + +// NewLogClient builds a LogClient for the /v1/logs endpoint from cfg. +func NewLogClient(cfg Config) (*LogClient, error) { + t, err := newRawTransport(cfg, pathLogs, nil) + if err != nil { + return nil, err + } + return &LogClient{t: t}, nil +} + +// ExportLogs posts each request atomically. It returns a non-nil error if any +// request failed; per-request detail is in the result. +func (c *LogClient) ExportLogs(ctx context.Context, requests []*logspb.ExportLogsServiceRequest) (*ExportResult, error) { + return exportEach(ctx, c.t, requests, logPartialSuccess) +} + +func logPartialSuccess(body []byte) (int64, string, error) { + var resp logspb.ExportLogsServiceResponse + if err := proto.Unmarshal(body, &resp); err != nil { + return 0, "", err + } + ps := resp.GetPartialSuccess() + return ps.GetRejectedLogRecords(), ps.GetErrorMessage(), nil +} diff --git a/otlp/export/metric.go b/otlp/export/metric.go new file mode 100644 index 0000000000..4d9849a5be --- /dev/null +++ b/otlp/export/metric.go @@ -0,0 +1,50 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package export + +import ( + "context" + + "google.golang.org/protobuf/proto" + + metricspb "go.opentelemetry.io/proto/otlp/collector/metrics/v1" +) + +// MetricClient exports offline OTLP metric requests. +type MetricClient struct { + t *rawTransport +} + +// NewMetricClient builds a MetricClient for the /v1/metrics endpoint from cfg. +// On the Datadog route it adds the dd-otel-metric-config header so exponential +// histograms are emitted as distributions. The header is not added on the +// collector/Agent route (Endpoint set), which does not understand it. +func NewMetricClient(cfg Config) (*MetricClient, error) { + var extra map[string]string + if cfg.Endpoint == "" && cfg.APIKey != "" { + extra = map[string]string{headerMetricConfig: metricConfigDistributions} + } + t, err := newRawTransport(cfg, pathMetrics, extra) + if err != nil { + return nil, err + } + return &MetricClient{t: t}, nil +} + +// ExportMetrics posts each request atomically. It returns a non-nil error if any +// request failed; per-request detail is in the result. +func (c *MetricClient) ExportMetrics(ctx context.Context, requests []*metricspb.ExportMetricsServiceRequest) (*ExportResult, error) { + return exportEach(ctx, c.t, requests, metricPartialSuccess) +} + +func metricPartialSuccess(body []byte) (int64, string, error) { + var resp metricspb.ExportMetricsServiceResponse + if err := proto.Unmarshal(body, &resp); err != nil { + return 0, "", err + } + ps := resp.GetPartialSuccess() + return ps.GetRejectedDataPoints(), ps.GetErrorMessage(), nil +} diff --git a/otlp/export/result.go b/otlp/export/result.go new file mode 100644 index 0000000000..1d769fc788 --- /dev/null +++ b/otlp/export/result.go @@ -0,0 +1,45 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package export + +// ExportResult reports the outcome of an ExportTraces/ExportMetrics/ExportLogs +// call. Each input request maps to exactly one RequestResult, by index. +type ExportResult struct { + Requests []RequestResult +} + +// RequestResult reports the outcome of a single request's POST. +type RequestResult struct { + // Index is the position of this request in the input slice. + Index int + // StatusCode is the final HTTP status code (0 if no response was received). + StatusCode int + // Attempts is the number of HTTP attempts made, including retries. + Attempts int + // Retriable reports whether the failure class was transient. It is only + // meaningful when Err is non-nil. + Retriable bool + // ResponseSnippet is a bounded, UTF-8-safe excerpt of the response body. + ResponseSnippet string + // Err is the error for this request, or nil on success. + Err error +} + +// Failed returns the number of requests that did not succeed. +func (r *ExportResult) Failed() int { + n := 0 + for _, req := range r.Requests { + if req.Err != nil { + n++ + } + } + return n +} + +// OK reports whether every request succeeded. +func (r *ExportResult) OK() bool { + return r.Failed() == 0 +} diff --git a/otlp/export/trace.go b/otlp/export/trace.go new file mode 100644 index 0000000000..e94bad75f2 --- /dev/null +++ b/otlp/export/trace.go @@ -0,0 +1,43 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package export + +import ( + "context" + + "google.golang.org/protobuf/proto" + + tracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" +) + +// TraceClient exports offline OTLP trace requests. +type TraceClient struct { + t *rawTransport +} + +// NewTraceClient builds a TraceClient for the /v1/traces endpoint from cfg. +func NewTraceClient(cfg Config) (*TraceClient, error) { + t, err := newRawTransport(cfg, pathTraces, nil) + if err != nil { + return nil, err + } + return &TraceClient{t: t}, nil +} + +// ExportTraces posts each request atomically. It returns a non-nil error if any +// request failed; per-request detail is in the result. +func (c *TraceClient) ExportTraces(ctx context.Context, requests []*tracepb.ExportTraceServiceRequest) (*ExportResult, error) { + return exportEach(ctx, c.t, requests, tracePartialSuccess) +} + +func tracePartialSuccess(body []byte) (int64, string, error) { + var resp tracepb.ExportTraceServiceResponse + if err := proto.Unmarshal(body, &resp); err != nil { + return 0, "", err + } + ps := resp.GetPartialSuccess() + return ps.GetRejectedSpans(), ps.GetErrorMessage(), nil +} diff --git a/otlp/export/transport.go b/otlp/export/transport.go new file mode 100644 index 0000000000..8901c2cd86 --- /dev/null +++ b/otlp/export/transport.go @@ -0,0 +1,282 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package export + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "google.golang.org/protobuf/encoding/protowire" + "google.golang.org/protobuf/proto" + + "github.com/DataDog/dd-trace-go/v2/internal/exportutil" +) + +const defaultRequestTimeout = 10 * time.Second + +var errNilRequest = errors.New("otlp/export: nil request") + +// rawTransport posts protobuf-encoded OTLP payloads to a fixed endpoint with +// bounded retry. +type rawTransport struct { + client *http.Client + endpoint string + headers http.Header + maxAttempts uint + requestTimeout time.Duration // 0 = default only when the caller sets no deadline +} + +// newRawTransport resolves the endpoint and headers for a signal and builds a +// transport. signalPath is one of the /v1/ constants; extraHeaders are +// signal-specific headers (e.g. the metric config) applied before Config.Headers. +func newRawTransport(cfg Config, signalPath string, extraHeaders map[string]string) (*rawTransport, error) { + base := cfg.Endpoint + if base == "" { + site := cfg.Site + if site == "" { + site = defaultSite + } + if cfg.APIKey == "" { + return nil, errors.New("otlp/export: APIKey is required for the Datadog OTLP route; set Endpoint to use a collector/Agent") + } + base = "https://otlp." + site + } + if u, err := url.Parse(base); err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return nil, fmt.Errorf("otlp/export: invalid endpoint %q: must be an http(s) URL with a host (OTLP/gRPC is not supported)", base) + } + endpoint := strings.TrimRight(base, "/") + signalPath + + // Assemble on an http.Header so keys are canonicalized and later layers + // deterministically override earlier ones regardless of caller casing. + headers := http.Header{} + headers.Set(headerContentType, contentTypeProto) + if cfg.APIKey != "" { + headers.Set(headerAPIKey, cfg.APIKey) + } + for k, v := range extraHeaders { + headers.Set(k, v) + } + for k, v := range cfg.Headers { + headers.Set(k, v) + } + + client := cfg.HTTPClient + if client == nil { + client = defaultHTTPClient() + } else if client.CheckRedirect == nil { + // Enforce no-redirect even on a caller-provided client that does not set its + // own policy: following a redirect would drop the POST body (a lost export + // reported as success) and forward the dd-api-key header to the redirect + // target. Copy the client so the caller's value is not mutated. + cp := *client + cp.CheckRedirect = noRedirect + client = &cp + } + maxAttempts := cfg.MaxAttempts + if maxAttempts == 0 { + maxAttempts = defaultMaxAttempts + } + return &rawTransport{client: client, endpoint: endpoint, headers: headers, maxAttempts: maxAttempts, requestTimeout: cfg.RequestTimeout}, nil +} + +// export marshals msg and POSTs it with bounded retry, returning a RequestResult +// and the raw response body (for partial-success decoding by the caller). +func (t *rawTransport) export(ctx context.Context, msg proto.Message) (RequestResult, []byte) { + var rr RequestResult + body, err := proto.Marshal(msg) + if err != nil { + rr.Err = fmt.Errorf("otlp/export: marshal: %w", err) + return rr, nil + } + res, err := exportutil.Retry(ctx, exportutil.RetryOptions{ + MaxAttempts: t.maxAttempts, + Retriable: otlpRetriable, + }, func(ctx context.Context) exportutil.Attempt { + return t.doPost(ctx, body) + }) + rr.StatusCode = res.StatusCode + rr.Attempts = res.Attempts + rr.Retriable = res.Retriable + rr.Err = err + // On failure an OTLP/HTTP endpoint returns a google.rpc.Status protobuf; + // surface its message rather than raw protobuf control bytes, falling back to + // the raw body when it is not a decodable Status. Either way the result runs + // through Snippet so ResponseSnippet stays bounded and UTF-8-safe. + if statusMsg := decodedStatusSnippet(err, res.Body); statusMsg != "" { + rr.ResponseSnippet = statusMsg + } else { + rr.ResponseSnippet = exportutil.Snippet(res.Body) + } + return rr, res.Body +} + +// decodedStatusSnippet returns a bounded snippet of the google.rpc.Status message +// for a failed request, or "" to fall back to a raw-body snippet. +func decodedStatusSnippet(err error, body []byte) string { + if err == nil { + return "" + } + return exportutil.Snippet([]byte(otlpStatusMessage(body))) +} + +func (t *rawTransport) doPost(ctx context.Context, body []byte) exportutil.Attempt { + // Apply a per-request timeout. An explicit Config.RequestTimeout always wins; + // otherwise fall back to the default only when the caller's context carries no + // deadline of its own, so a caller that passes a longer deadline is not + // silently shortened. + if t.requestTimeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, t.requestTimeout) + defer cancel() + } else if _, ok := ctx.Deadline(); !ok { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, defaultRequestTimeout) + defer cancel() + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, t.endpoint, bytes.NewReader(body)) + if err != nil { + return exportutil.Attempt{Err: err} + } + req.Header = t.headers.Clone() + + resp, err := t.client.Do(req) + if err != nil { + return exportutil.Attempt{Err: err} + } + defer resp.Body.Close() + + respBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + // OTLP/HTTP defines success as exactly 200 OK with a protobuf Export*Response + // body. Treat any other 2xx (202/204/206, or a redirect not followed) as a + // failed export rather than silently reporting delivery; the body decode is + // validated by the caller's partial-success decoder. + if resp.StatusCode == http.StatusOK { + if readErr != nil { + // The 200 body carries OTLP partial-success rejections; a failed read + // could hide dropped records, so surface it as a transport-class + // (retryable, status 0) failure instead of reporting full success. + return exportutil.Attempt{Body: respBody, Err: fmt.Errorf("otlp/export: read response body: %w", readErr)} + } + return exportutil.Attempt{Status: resp.StatusCode, Body: respBody} + } + return exportutil.Attempt{ + Status: resp.StatusCode, + Body: respBody, + RetryAfter: parseRetryAfter(resp.Header), + Err: fmt.Errorf("otlp/export: unexpected status %d", resp.StatusCode), + } +} + +// otlpRetriable applies the OTLP/HTTP retryable-status rules: only 429, 502, 503 +// and 504 (plus network-level errors, status 0) are retried; every other 4xx/5xx +// is permanent. See +// https://opentelemetry.io/docs/specs/otlp/#retryable-response-codes. +func otlpRetriable(ctx context.Context, status int) bool { + if ctx.Err() != nil { + return false + } + switch status { + case 0, http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout: + return true + default: + return false + } +} + +const maxRetryAfter = 60 * time.Second + +// parseRetryAfter reads a Retry-After header (delta-seconds or HTTP-date) and +// returns the delay to wait, clamped to (0, maxRetryAfter]. It returns 0 when the +// header is absent or unparseable so the caller falls back to backoff. +func parseRetryAfter(h http.Header) time.Duration { + v := strings.TrimSpace(h.Get("Retry-After")) + if v == "" { + return 0 + } + if secs, err := strconv.ParseInt(v, 10, 64); err == nil { + if secs <= 0 { + return 0 + } + // Parse as int64 (not int, which is 32-bit on some builds) and clamp the + // second-count before converting, so a large hint (e.g. "9223372037") is + // capped rather than overflowing time.Duration into a non-positive value. + if secs > int64(maxRetryAfter/time.Second) { + return maxRetryAfter + } + return time.Duration(secs) * time.Second + } + if t, err := http.ParseTime(v); err == nil { + return clampRetryAfter(time.Until(t)) + } + return 0 +} + +func clampRetryAfter(d time.Duration) time.Duration { + switch { + case d <= 0: + return 0 + case d > maxRetryAfter: + return maxRetryAfter + default: + return d + } +} + +// otlpStatusMessage best-effort extracts the human-readable message from a +// google.rpc.Status protobuf body (field 2, a string). It returns "" when body +// is not a decodable Status, without pulling in the generated Status type. +func otlpStatusMessage(body []byte) string { + b := body + for len(b) > 0 { + num, typ, n := protowire.ConsumeTag(b) + if n < 0 { + return "" + } + b = b[n:] + if num == 2 && typ == protowire.BytesType { + v, vn := protowire.ConsumeBytes(b) + if vn < 0 { + return "" + } + return string(v) + } + skip := protowire.ConsumeFieldValue(num, typ, b) + if skip < 0 { + return "" + } + b = b[skip:] + } + return "" +} + +// noRedirect stops the HTTP client from following redirects: Go would replay the +// POST as a GET with the body dropped (a lost export reported as success) and +// forward the dd-api-key header to the redirect target (a credential leak). +// Surfacing the 3xx as a non-200 response makes it a failed export instead. +func noRedirect(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + +func defaultHTTPClient() *http.Client { + return &http.Client{ + CheckRedirect: noRedirect, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: time.Second, + }, + } +} diff --git a/otlp/export/transport_internal_test.go b/otlp/export/transport_internal_test.go new file mode 100644 index 0000000000..17b011cad8 --- /dev/null +++ b/otlp/export/transport_internal_test.go @@ -0,0 +1,217 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2026 Datadog, Inc. + +package export + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" +) + +// stubRoundTripper returns a canned response (or error) without a network. +type stubRoundTripper struct { + status int + header http.Header + body io.ReadCloser + err error +} + +func (s stubRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + if s.err != nil { + return nil, s.err + } + h := s.header + if h == nil { + h = http.Header{} + } + return &http.Response{StatusCode: s.status, Header: h, Body: s.body}, nil +} + +// errReadCloser fails on the first Read, simulating a reset connection after the +// status/headers were received. +type errReadCloser struct{} + +func (errReadCloser) Read([]byte) (int, error) { return 0, errors.New("connection reset") } +func (errReadCloser) Close() error { return nil } + +func stubTransport(rt http.RoundTripper) *rawTransport { + return &rawTransport{client: &http.Client{Transport: rt}, endpoint: "http://x/v1/traces", headers: http.Header{}, maxAttempts: 1} +} + +func TestDoPost_ReadErrorOn2xxIsSurfacedAsRetryableFailure(t *testing.T) { + tr := stubTransport(stubRoundTripper{status: 200, body: errReadCloser{}}) + a := tr.doPost(context.Background(), []byte("payload")) + require.Error(t, a.Err) // an unreadable 2xx body must not be reported as success + assert.Equal(t, 0, a.Status) + assert.True(t, otlpRetriable(context.Background(), a.Status)) // treated as transport-class -> retryable +} + +func TestDefaultHTTPClient_DoesNotFollowRedirects(t *testing.T) { + c := defaultHTTPClient() + require.NotNil(t, c.CheckRedirect) // must not use Go's default follow-redirect policy + // A redirect must surface the 3xx instead of replaying the POST as a GET + // (dropped body) or forwarding dd-api-key to the redirect target. + assert.ErrorIs(t, c.CheckRedirect(nil, nil), http.ErrUseLastResponse) +} + +func TestNewRawTransport_EnforcesNoRedirectOnCustomClient(t *testing.T) { + // A caller-provided client without its own redirect policy gets no-redirect + // enforced (so dd-api-key is never forwarded and a redirect is a failed export). + custom := &http.Client{} + tr, err := newRawTransport(Config{Site: "datadoghq.com", APIKey: "k", HTTPClient: custom}, pathTraces, nil) + require.NoError(t, err) + require.NotNil(t, tr.client.CheckRedirect) + assert.ErrorIs(t, tr.client.CheckRedirect(nil, nil), http.ErrUseLastResponse) + assert.Nil(t, custom.CheckRedirect, "caller's client must not be mutated") + + // A caller that sets its own redirect policy is respected, not overridden. + sentinel := errors.New("caller policy") + custom2 := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return sentinel }} + tr2, err := newRawTransport(Config{Site: "datadoghq.com", APIKey: "k", HTTPClient: custom2}, pathTraces, nil) + require.NoError(t, err) + assert.ErrorIs(t, tr2.client.CheckRedirect(nil, nil), sentinel) +} + +// deadlineCapture records the deadline seen by the outgoing request's context. +type deadlineCapture struct { + dur func(remaining time.Duration, ok bool) +} + +func (d deadlineCapture) RoundTrip(req *http.Request) (*http.Response, error) { + dl, ok := req.Context().Deadline() + rem := time.Duration(0) + if ok { + rem = time.Until(dl) + } + d.dur(rem, ok) + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader("")), Header: http.Header{}}, nil +} + +func TestDoPost_RequestTimeout(t *testing.T) { + var rem time.Duration + var hasDeadline bool + rt := deadlineCapture{dur: func(r time.Duration, ok bool) { rem, hasDeadline = r, ok }} + mk := func(reqTimeout time.Duration) *rawTransport { + return &rawTransport{client: &http.Client{Transport: rt}, endpoint: "http://x/v1/traces", headers: http.Header{}, maxAttempts: 1, requestTimeout: reqTimeout} + } + + // Explicit RequestTimeout is applied to the attempt. + mk(5*time.Second).doPost(context.Background(), []byte("x")) + require.True(t, hasDeadline) + assert.InDelta(t, 5.0, rem.Seconds(), 1.0) + + // RequestTimeout==0, no caller deadline -> the 10s default is applied. + mk(0).doPost(context.Background(), []byte("x")) + require.True(t, hasDeadline) + assert.InDelta(t, 10.0, rem.Seconds(), 1.0) + + // RequestTimeout==0 with a longer caller deadline -> respected, not shortened. + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + mk(0).doPost(ctx, []byte("x")) + require.True(t, hasDeadline) + assert.Greater(t, rem.Seconds(), 20.0) +} + +func TestDoPost_Non200IsFailure(t *testing.T) { + // 202/204/206 are 2xx but not the OTLP success contract (200 + proto body); + // a not-followed redirect (302) must also be a failure. + for _, code := range []int{202, 204, 206, 302} { + tr := stubTransport(stubRoundTripper{status: code, body: io.NopCloser(strings.NewReader(""))}) + a := tr.doPost(context.Background(), []byte("payload")) + require.Errorf(t, a.Err, "status %d should be a failed export", code) + assert.Equal(t, code, a.Status) + } +} + +func TestDoPost_ThreadsRetryAfterHeader(t *testing.T) { + tr := stubTransport(stubRoundTripper{ + status: 503, + header: http.Header{"Retry-After": []string{"2"}}, + body: io.NopCloser(strings.NewReader("busy")), + }) + a := tr.doPost(context.Background(), []byte("payload")) + require.Error(t, a.Err) + assert.Equal(t, 503, a.Status) + assert.Equal(t, 2*time.Second, a.RetryAfter) // header parsed and threaded into the Attempt +} + +func TestOTLPRetriable(t *testing.T) { + ctx := context.Background() + // Per the OTLP/HTTP spec only 429, 502, 503, 504 (and network errors) retry. + for status, want := range map[int]bool{ + 0: true, // network-level error + 429: true, + 502: true, + 503: true, + 504: true, + 408: false, // retryable under the generic classifier, but not OTLP + 500: false, + 501: false, + 505: false, + 400: false, + 404: false, + 200: false, + } { + assert.Equalf(t, want, otlpRetriable(ctx, status), "status %d", status) + } + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + assert.False(t, otlpRetriable(cancelled, 503)) // a cancelled context is never retriable +} + +func TestParseRetryAfter(t *testing.T) { + mk := func(v string) http.Header { return http.Header{"Retry-After": []string{v}} } + + assert.Equal(t, time.Duration(0), parseRetryAfter(http.Header{})) // absent + assert.Equal(t, time.Duration(0), parseRetryAfter(mk(""))) // empty + assert.Equal(t, 5*time.Second, parseRetryAfter(mk("5"))) // delta-seconds + assert.Equal(t, 10*time.Second, parseRetryAfter(mk(" 10 "))) // trimmed + assert.Equal(t, time.Duration(0), parseRetryAfter(mk("-3"))) // negative clamps to 0 + assert.Equal(t, maxRetryAfter, parseRetryAfter(mk("100000"))) // clamped to the cap + assert.Equal(t, maxRetryAfter, parseRetryAfter(mk("9223372037"))) // overflow value clamped, not wrapped negative + assert.Equal(t, time.Duration(0), parseRetryAfter(mk("soon"))) // unparseable + assert.Equal(t, time.Duration(0), parseRetryAfter(mk("0"))) // zero -> fall back to backoff + + // HTTP-date in the future yields a positive, capped delay. + future := time.Now().Add(3 * time.Second).UTC().Format(http.TimeFormat) + d := parseRetryAfter(mk(future)) + assert.Greater(t, d, time.Duration(0)) + assert.LessOrEqual(t, d, maxRetryAfter) + + // HTTP-date in the past clamps to 0. + past := time.Now().Add(-time.Hour).UTC().Format(http.TimeFormat) + assert.Equal(t, time.Duration(0), parseRetryAfter(mk(past))) +} + +func TestOTLPStatusMessage(t *testing.T) { + // A google.rpc.Status: field 1 = code (varint), field 2 = message (string). + var body []byte + body = protowire.AppendTag(body, 1, protowire.VarintType) + body = protowire.AppendVarint(body, 3) // INVALID_ARGUMENT + body = protowire.AppendTag(body, 2, protowire.BytesType) + body = protowire.AppendBytes(body, []byte("invalid trace_id length")) + assert.Equal(t, "invalid trace_id length", otlpStatusMessage(body)) + + // Missing message field -> "". + var codeOnly []byte + codeOnly = protowire.AppendTag(codeOnly, 1, protowire.VarintType) + codeOnly = protowire.AppendVarint(codeOnly, 5) + assert.Equal(t, "", otlpStatusMessage(codeOnly)) + + // Non-protobuf garbage -> "" (best-effort, never panics). + assert.Equal(t, "", otlpStatusMessage([]byte{0xff, 0xff, 0xff})) + assert.Equal(t, "", otlpStatusMessage(nil)) +}