From db1fbc2d46047621825a65b36f15e7372b31404e Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 23 Mar 2026 09:21:16 -0400 Subject: [PATCH 01/18] wip: default value for failed serialization --- ddtrace/tracer/payload_test.go | 73 ++++++++++++++++++++++++++++++++++ ddtrace/tracer/payload_v1.go | 37 +++++++++++++---- 2 files changed, 102 insertions(+), 8 deletions(-) diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index f13001b911e..687516546e7 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -532,6 +532,79 @@ func assertProcessTags(t *testing.T, payload spanLists) { } } +func TestPayloadV1SerializationFailure(t *testing.T) { + t.Run("nil span", func(t *testing.T) { + assert := assert.New(t) + p := newPayloadV1() + sl := newSpanList(1) + sl = append(sl, nil) // add a nil span + + _, err := p.push(sl) + assert.NoError(err) + + encoded, err := io.ReadAll(p) + assert.NoError(err) + + got := newPayloadV1() + buf := bytes.NewBuffer(encoded) + _, err = buf.WriteTo(got) + assert.NoError(err) + + _, err = got.decodeBuffer() + assert.NoError(err) + + require.Len(t, got.chunks, 1) + require.Len(t, got.chunks[0].spans, 2) + assert.Equal(&Span{}, got.chunks[0].spans[1]) + }) + + t.Run("unsupported type conversions", func(t *testing.T) { + assert := assert.New(t) + p := newPayloadV1() + + s := newBasicSpan("test.span") + s.setMetaStructLocked("bad-key", make(chan int)) // unsupported type + _, err := p.push(spanList{s}) + assert.NoError(err) + + encoded, err := io.ReadAll(p) + assert.NoError(err) + + got := newPayloadV1() + buf := bytes.NewBuffer(encoded) + _, err = buf.WriteTo(got) + assert.NoError(err) + + _, err = got.decodeBuffer() + assert.NoError(err) + + require.Len(t, got.chunks, 1) + require.Len(t, got.chunks[0].spans, 1) + span := got.chunks[0].spans[0] + assert.Equal(serializationFailed, span.meta["bad-key"]) + }) + + t.Run("invalid valueType", func(t *testing.T) { + p := newPayloadV1() + p.attributes["bad-attr"] = anyValue{valueType: 999, value: "x"} + s := newBasicSpan("test-span") + _, err := p.push(spanList{s}) + require.NoError(t, err) + encoded, err := io.ReadAll(p) + require.NoError(t, err) + + got := newPayloadV1() + _, err = bytes.NewBuffer(encoded).WriteTo(got) + require.NoError(t, err) + + _, err = got.decodeBuffer() + require.NoError(t, err) + require.NotNil(t, got.attributes["bad-attr"]) + assert.Equal(t, StringValueType, got.attributes["bad-attr"].valueType) + assert.Equal(t, serializationFailed, got.attributes["bad-attr"].value) + }) +} + func BenchmarkPayloadThroughput(b *testing.B) { b.Run("10K", benchmarkPayloadThroughput(1)) b.Run("100K", benchmarkPayloadThroughput(10)) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 520dac1a10c..a2534205dbd 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -119,6 +119,10 @@ type payloadV1 struct { processTagsStr string } +// Constant dummy value to represent a serialization failure. Used to prevent failures while +// decoding the payload. +const serializationFailed string = "serialization_failed" + // newPayloadV1 returns a ready to use payloadV1. func newPayloadV1() *payloadV1 { return &payloadV1{ @@ -455,6 +459,9 @@ func encodeField[F fieldValue](buf []byte, bm bitmap, fieldID uint32, a F, st *s for _, v := range value { buf = v.encode(buf, st) } + default: + log.Warn("failed to serialize value: %v", value) + buf = st.serialize(serializationFailed, buf) } return buf } @@ -556,6 +563,8 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri var scratch []byte for _, span := range spans { if span == nil { + // encode an empty map for nil spans + p.buf = msgp.AppendMapHeader(p.buf, 0) continue } p.buf = msgp.AppendMapHeader(p.buf, 16) // number of fields in span @@ -589,7 +598,10 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri var err error scratch, err = msgp.AppendIntf(scratch[:0], v) if err != nil { - log.Error("failed to serialize meta_struct value for key %s: %v", k, err.Error()) + log.Warn("failed to serialize meta_struct value for key %s: %v", k, err.Error()) + p.buf = st.serialize(k, p.buf) + p.buf = msgp.AppendUint32(p.buf, uint32(StringValueType)) + p.buf = st.serialize(serializationFailed, p.buf) continue } p.buf = st.serialize(k, p.buf) @@ -701,25 +713,21 @@ func (p *payloadV1) encodeSpanEvents(bm bitmap, fieldID int, spanEvents []spanEv p.buf = msgp.AppendUint32(p.buf, uint32(3)) // attributes fieldID p.buf = msgp.AppendArrayHeader(p.buf, uint32(len(event.Attributes))*3) // number of attributes for k, v := range event.Attributes { + p.buf = st.serialize(k, p.buf) switch v.Type { case spanEventAttributeTypeString: - p.buf = st.serialize(k, p.buf) p.buf = msgp.AppendUint32(p.buf, uint32(StringValueType)) p.buf = st.serialize(v.StringValue, p.buf) case spanEventAttributeTypeInt: - p.buf = st.serialize(k, p.buf) p.buf = msgp.AppendUint32(p.buf, uint32(IntValueType)) p.buf = msgp.AppendInt64(p.buf, v.IntValue) case spanEventAttributeTypeDouble: - p.buf = st.serialize(k, p.buf) p.buf = msgp.AppendUint32(p.buf, uint32(FloatValueType)) p.buf = msgp.AppendFloat64(p.buf, v.DoubleValue) case spanEventAttributeTypeBool: - p.buf = st.serialize(k, p.buf) p.buf = msgp.AppendUint32(p.buf, uint32(BoolValueType)) p.buf = msgp.AppendBool(p.buf, v.BoolValue) case spanEventAttributeTypeArray: - p.buf = st.serialize(k, p.buf) p.buf = msgp.AppendUint32(p.buf, uint32(ArrayValueType)) // Array format is (type, value) per element; decoder expects len/2 anyValues. p.buf = msgp.AppendArrayHeader(p.buf, uint32(len(v.ArrayValue.Values))*2) @@ -728,6 +736,8 @@ func (p *payloadV1) encodeSpanEvents(bm bitmap, fieldID int, spanEvents []spanEv } default: log.Warn("dropped unsupported span event attribute type %d", v.Type) + p.buf = msgp.AppendUint32(p.buf, uint32(StringValueType)) + p.buf = st.serialize(serializationFailed, p.buf) } } } @@ -749,7 +759,9 @@ func (p *payloadV1) encodeSpanEventArrayValues(v *spanEventArrayAttributeValue, p.buf = msgp.AppendUint32(p.buf, uint32(BoolValueType)) p.buf = msgp.AppendBool(p.buf, v.BoolValue) default: - log.Warn("dropped unsupported span event array attribute type %d", v.Type) + log.Warn("could not serialize unsupported span event array attribute type %d", v.Type) + p.buf = msgp.AppendUint32(p.buf, uint32(StringValueType)) + p.buf = st.serialize(serializationFailed, p.buf) } return true, nil } @@ -914,24 +926,33 @@ const ( ) func (a anyValue) encode(buf []byte, st *stringTable) []byte { - buf = msgp.AppendInt32(buf, int32(a.valueType)) switch a.valueType { case StringValueType: + buf = msgp.AppendInt32(buf, int32(a.valueType)) s := a.value.(string) buf = st.serialize(s, buf) case BoolValueType: + buf = msgp.AppendInt32(buf, int32(a.valueType)) buf = msgp.AppendBool(buf, a.value.(bool)) case FloatValueType: + buf = msgp.AppendInt32(buf, int32(a.valueType)) buf = msgp.AppendFloat64(buf, a.value.(float64)) case IntValueType: + buf = msgp.AppendInt32(buf, int32(a.valueType)) buf = msgp.AppendInt64(buf, a.value.(int64)) case BytesValueType: + buf = msgp.AppendInt32(buf, int32(a.valueType)) buf = msgp.AppendBytes(buf, a.value.([]byte)) case ArrayValueType: + buf = msgp.AppendInt32(buf, int32(a.valueType)) buf = msgp.AppendArrayHeader(buf, uint32(len(a.value.(arrayValue)))) for _, v := range a.value.(arrayValue) { buf = v.encode(buf, st) } + default: + log.Warn("failed to serialize value: %v", a.value) + buf = msgp.AppendInt32(buf, StringValueType) + buf = st.serialize(serializationFailed, buf) } return buf } From 44211b7eccaada17ae11418257f36c215117f520 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 23 Mar 2026 15:06:30 -0400 Subject: [PATCH 02/18] write dummy value as a byte for meta_struct --- ddtrace/tracer/payload_test.go | 26 ++++++++++++++++++++++++++ ddtrace/tracer/payload_v1.go | 5 +---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index 687516546e7..9d5beca4331 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -603,6 +603,32 @@ func TestPayloadV1SerializationFailure(t *testing.T) { assert.Equal(t, StringValueType, got.attributes["bad-attr"].valueType) assert.Equal(t, serializationFailed, got.attributes["bad-attr"].value) }) + + t.Run("invalid meta struct value", func(t *testing.T) { + p := newPayloadV1() + s := newBasicSpan("test-span") + s.mu.Lock() + s.setMetaStructLocked("bad-key", make(chan int)) // unsupported type + s.mu.Unlock() + _, err := p.push(spanList{s}) + require.NoError(t, err) + encoded, err := io.ReadAll(p) + require.NoError(t, err) + + got := newPayloadV1() + _, err = bytes.NewBuffer(encoded).WriteTo(got) + require.NoError(t, err) + + _, err = got.decodeBuffer() + require.NoError(t, err) + require.Len(t, got.chunks, 1) + require.Len(t, got.chunks[0].spans, 1) + ms := got.chunks[0].spans[0].metaStruct["bad-key"] + require.NotNil(t, ms) + v, ok := ms.([]byte) + assert.True(t, ok) + assert.Equal(t, []byte(serializationFailed), v) + }) } func BenchmarkPayloadThroughput(b *testing.B) { diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index a2534205dbd..7e66bd640b8 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -599,10 +599,7 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri scratch, err = msgp.AppendIntf(scratch[:0], v) if err != nil { log.Warn("failed to serialize meta_struct value for key %s: %v", k, err.Error()) - p.buf = st.serialize(k, p.buf) - p.buf = msgp.AppendUint32(p.buf, uint32(StringValueType)) - p.buf = st.serialize(serializationFailed, p.buf) - continue + scratch, _ = msgp.AppendIntf(nil, []byte(serializationFailed)) } p.buf = st.serialize(k, p.buf) p.buf = msgp.AppendUint32(p.buf, uint32(BytesValueType)) From f3ac9a2900ab003e32373c50c06e2a3111a14523 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 23 Mar 2026 15:16:27 -0400 Subject: [PATCH 03/18] dont double encode span links on spans --- ddtrace/tracer/payload_test.go | 29 +++-------------------------- ddtrace/tracer/payload_v1.go | 5 +++++ 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index 9d5beca4331..ff312b2c9b9 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -315,6 +315,9 @@ func TestPayloadV1SpanLinkTraceID(t *testing.T) { assert.Equal(uint64(123), link.TraceID) assert.Equal(uint64(456), link.TraceIDHigh) assert.Equal(uint64(789), link.SpanID) + + span = got.chunks[0].spans[0] + assert.Empty(span.meta["_dd.span_links"]) } // TestPayloadV1SpanEventArray tests that a span with a span event containing ArrayValue @@ -558,32 +561,6 @@ func TestPayloadV1SerializationFailure(t *testing.T) { assert.Equal(&Span{}, got.chunks[0].spans[1]) }) - t.Run("unsupported type conversions", func(t *testing.T) { - assert := assert.New(t) - p := newPayloadV1() - - s := newBasicSpan("test.span") - s.setMetaStructLocked("bad-key", make(chan int)) // unsupported type - _, err := p.push(spanList{s}) - assert.NoError(err) - - encoded, err := io.ReadAll(p) - assert.NoError(err) - - got := newPayloadV1() - buf := bytes.NewBuffer(encoded) - _, err = buf.WriteTo(got) - assert.NoError(err) - - _, err = got.decodeBuffer() - assert.NoError(err) - - require.Len(t, got.chunks, 1) - require.Len(t, got.chunks[0].spans, 1) - span := got.chunks[0].spans[0] - assert.Equal(serializationFailed, span.meta["bad-key"]) - }) - t.Run("invalid valueType", func(t *testing.T) { p := newPayloadV1() p.attributes["bad-attr"] = anyValue{valueType: 999, value: "x"} diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 7e66bd640b8..37e71f1bf86 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -585,6 +585,11 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri p.buf = msgp.AppendUint32(p.buf, uint32(9)) // attributes fieldID p.buf = msgp.AppendArrayHeader(p.buf, uint32(size)*3) // number of attributes for k, v := range span.meta { + // Span links are serialized separately in the payload, so + // we skip them here to avoid duplication. + if k == "_dd.span_links" { + continue + } p.buf = st.serialize(k, p.buf) p.buf = msgp.AppendUint32(p.buf, uint32(StringValueType)) p.buf = st.serialize(v, p.buf) From fef304ec2e60e1d8983f25bbad87f20ce9f0a60d Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Tue, 24 Mar 2026 10:24:25 -0400 Subject: [PATCH 04/18] fix: reduce heap allocs? --- ddtrace/tracer/payload_v1.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 37e71f1bf86..bcf3aa4c10f 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -460,7 +460,7 @@ func encodeField[F fieldValue](buf []byte, bm bitmap, fieldID uint32, a F, st *s buf = v.encode(buf, st) } default: - log.Warn("failed to serialize value: %v", value) + log.Warn("failed to serialize unsupported fieldValue type for field %d", fieldID) buf = st.serialize(serializationFailed, buf) } return buf @@ -603,7 +603,7 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri var err error scratch, err = msgp.AppendIntf(scratch[:0], v) if err != nil { - log.Warn("failed to serialize meta_struct value for key %s: %v", k, err.Error()) + log.Warn("failed to serialize meta_struct value for key %s: %s", k, err.Error()) scratch, _ = msgp.AppendIntf(nil, []byte(serializationFailed)) } p.buf = st.serialize(k, p.buf) From 5a8e764fa6204f580db8aecf26dd97b32eb6f4fb Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Tue, 24 Mar 2026 13:21:33 -0400 Subject: [PATCH 05/18] trigger tests From f81c6ec6cedfcec79806cc383f2a46b418fc2978 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Tue, 24 Mar 2026 13:27:08 -0400 Subject: [PATCH 06/18] lint: don't use %v --- ddtrace/tracer/payload_v1.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index bcf3aa4c10f..fcc11d7c2c3 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -952,7 +952,7 @@ func (a anyValue) encode(buf []byte, st *stringTable) []byte { buf = v.encode(buf, st) } default: - log.Warn("failed to serialize value: %v", a.value) + log.Warn("failed to serialize value type: %d", a.valueType) buf = msgp.AppendInt32(buf, StringValueType) buf = st.serialize(serializationFailed, buf) } From ec8baf251edb12ca757abf93422c9939ae62dca0 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Tue, 24 Mar 2026 16:17:42 -0400 Subject: [PATCH 07/18] fix: array length with _dd.span_links was wrong --- ddtrace/tracer/payload_test.go | 2 ++ ddtrace/tracer/payload_v1.go | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index ff312b2c9b9..f791c061fe8 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -293,6 +293,8 @@ func TestPayloadV1SpanLinkTraceID(t *testing.T) { span.spanLinks = []SpanLink{ {TraceID: 123, TraceIDHigh: 456, SpanID: 789}, } + span.setMeta("_dd.span_links", "test") // should not get serialized + _, err := p.push(spanList{span}) assert.NoError(err) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index fcc11d7c2c3..29cc0106b7f 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -582,6 +582,10 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri // To avoid increased allocations, we serialize attributes immediately without // creating an intermediate map. size := len(span.meta) + len(span.metrics) + len(span.metaStruct) + _, containsSpanLinks := span.meta["_dd.span_links"] + if containsSpanLinks { + size-- + } p.buf = msgp.AppendUint32(p.buf, uint32(9)) // attributes fieldID p.buf = msgp.AppendArrayHeader(p.buf, uint32(size)*3) // number of attributes for k, v := range span.meta { From 682035bcc7255c0c61074e6c4570d042e04c2b71 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 30 Mar 2026 10:43:15 -0400 Subject: [PATCH 08/18] inlining fixes --- ddtrace/tracer/payload_v1.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 705d296a815..1a70017f232 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -572,8 +572,7 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri // To avoid increased allocations, we serialize attributes immediately without // creating an intermediate map. size := len(span.meta) + len(span.metrics) + len(span.metaStruct) - _, containsSpanLinks := span.meta["_dd.span_links"] - if containsSpanLinks { + if _, ok := span.meta["_dd.span_links"]; ok { size-- } p.buf = msgp.AppendUint32(p.buf, uint32(9)) // attributes fieldID @@ -731,7 +730,7 @@ func (p *payloadV1) encodeSpanEvents(bm bitmap, fieldID int, spanEvents []spanEv p.encodeSpanEventArrayValues(v, st) } default: - log.Warn("dropped unsupported span event attribute type %d", v.Type) + warnUnsupportedValue(uint32(v.Type)) p.buf = msgp.AppendUint32(p.buf, uint32(StringValueType)) p.buf = st.serialize(serializationFailed, p.buf) } @@ -755,7 +754,7 @@ func (p *payloadV1) encodeSpanEventArrayValues(v *spanEventArrayAttributeValue, p.buf = msgp.AppendUint32(p.buf, uint32(BoolValueType)) p.buf = msgp.AppendBool(p.buf, v.BoolValue) default: - log.Warn("could not serialize unsupported span event array attribute type %d", v.Type) + warnUnsupportedValue(uint32(v.Type)) p.buf = msgp.AppendUint32(p.buf, uint32(StringValueType)) p.buf = st.serialize(serializationFailed, p.buf) } @@ -946,7 +945,7 @@ func (a anyValue) encode(buf []byte, st *stringTable) []byte { buf = v.encode(buf, st) } default: - log.Warn("failed to serialize value type: %d", a.valueType) + warnUnsupportedValue(uint32(a.valueType)) buf = msgp.AppendInt32(buf, StringValueType) buf = st.serialize(serializationFailed, buf) } @@ -1009,7 +1008,6 @@ func (i *index) decode(buf []byte) ([]byte, error) { type stringValue string func (s stringValue) encode(buf []byte) []byte { - // TODO(hannahkm): add the fixstr representation return msgp.AppendString(buf, string(s)) } @@ -1610,3 +1608,8 @@ func decodeAttributes(b []byte, strings *stringTable) (map[string]anyValue, []by } return kv, o, nil } + +//go:noinline +func warnUnsupportedValue(t uint32) { + log.Warn("failed to serialize unsupported type: %d", t) +} From 6613e9f5757592dca1ad3099570b8e7c44269cf3 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 30 Mar 2026 11:27:22 -0400 Subject: [PATCH 09/18] attempt: write array header after contents to save time on reading span links --- ddtrace/tracer/payload_v1.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 1a70017f232..38827c02bce 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -570,24 +570,25 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri // span attributes combine the meta (tags), metrics and meta_struct // To avoid increased allocations, we serialize attributes immediately without - // creating an intermediate map. - size := len(span.meta) + len(span.metrics) + len(span.metaStruct) - if _, ok := span.meta["_dd.span_links"]; ok { - size-- - } - p.buf = msgp.AppendUint32(p.buf, uint32(9)) // attributes fieldID - p.buf = msgp.AppendArrayHeader(p.buf, uint32(size)*3) // number of attributes + // creating an intermediate map. We also write a placeholder for the array header + // and write the actual count after writing all attributes + p.buf = msgp.AppendUint32(p.buf, uint32(9)) // attributes fieldID + off := len(p.buf) + count := 0 + p.buf = append(p.buf, msgpackArray32, 0, 0, 0, 0) for k, v := range span.meta { // Span links are serialized separately in the payload, so // we skip them here to avoid duplication. if k == "_dd.span_links" { continue } + count++ p.buf = st.serialize(k, p.buf) p.buf = msgp.AppendUint32(p.buf, uint32(StringValueType)) p.buf = st.serialize(v, p.buf) } for k, v := range span.metrics { + count++ p.buf = st.serialize(k, p.buf) p.buf = msgp.AppendUint32(p.buf, uint32(FloatValueType)) p.buf = msgp.AppendFloat64(p.buf, v) @@ -599,11 +600,18 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri log.Warn("failed to serialize meta_struct value for key %s: %s", k, err.Error()) scratch, _ = msgp.AppendIntf(nil, []byte(serializationFailed)) } + count++ p.buf = st.serialize(k, p.buf) p.buf = msgp.AppendUint32(p.buf, uint32(BytesValueType)) p.buf = msgp.AppendBytes(p.buf, scratch) } + elementCount := uint32(count) * 3 + p.buf[off+1] = byte(elementCount >> 24) + p.buf[off+2] = byte(elementCount >> 16) + p.buf[off+3] = byte(elementCount >> 8) + p.buf[off+4] = byte(elementCount) + p.buf = encodeField(p.buf, fullSetBitmap, 10, span.spanType, st) p.encodeSpanLinks(fullSetBitmap, 11, span.spanLinks, st) p.encodeSpanEvents(fullSetBitmap, 12, span.spanEvents, st) From 0a7fffe578014c067024f1b8789d28089c717b26 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 30 Mar 2026 13:29:07 -0400 Subject: [PATCH 10/18] get meta values proactively --- ddtrace/tracer/payload_v1.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 38827c02bce..a77284720f7 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -576,12 +576,26 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri off := len(p.buf) count := 0 p.buf = append(p.buf, msgpackArray32, 0, 0, 0, 0) + env, version, component, spanKind := "", "", "", "" for k, v := range span.meta { // Span links are serialized separately in the payload, so // we skip them here to avoid duplication. if k == "_dd.span_links" { continue } + // Grab common attributes early to avoid map lookups later on. + if k == ext.Environment { + env = v + } + if k == ext.Version { + version = v + } + if k == ext.Component { + component = v + } + if k == ext.SpanKind { + spanKind = v + } count++ p.buf = st.serialize(k, p.buf) p.buf = msgp.AppendUint32(p.buf, uint32(StringValueType)) @@ -615,17 +629,9 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri p.buf = encodeField(p.buf, fullSetBitmap, 10, span.spanType, st) p.encodeSpanLinks(fullSetBitmap, 11, span.spanLinks, st) p.encodeSpanEvents(fullSetBitmap, 12, span.spanEvents, st) - - env := span.meta[ext.Environment] p.buf = encodeField(p.buf, fullSetBitmap, 13, env, st) - - version := span.meta[ext.Version] p.buf = encodeField(p.buf, fullSetBitmap, 14, version, st) - - component := span.meta[ext.Component] p.buf = encodeField(p.buf, fullSetBitmap, 15, component, st) - - spanKind := span.meta[ext.SpanKind] p.buf = encodeField(p.buf, fullSetBitmap, 16, getSpanKindValue(spanKind), st) } return true, nil From 91cecd6399d993f2235a8e53c16249f692953490 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Tue, 31 Mar 2026 13:32:36 -0400 Subject: [PATCH 11/18] only create string table if it doesn't already exist --- ddtrace/tracer/payload_v1.go | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index a77284720f7..c3edeaa2d0e 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -402,19 +402,21 @@ func (p *payloadV1) update() { // encode writes existing payload fields into the buffer in msgp format. func (p *payloadV1) encode() { - st := newStringTable() - p.buf = encodeField(p.buf, p.bm, 2, p.containerID, st) - p.buf = encodeField(p.buf, p.bm, 3, p.languageName, st) - p.buf = encodeField(p.buf, p.bm, 4, p.languageVersion, st) - p.buf = encodeField(p.buf, p.bm, 5, p.tracerVersion, st) - p.buf = encodeField(p.buf, p.bm, 6, p.runtimeID, st) - p.buf = encodeField(p.buf, p.bm, 7, p.env, st) - p.buf = encodeField(p.buf, p.bm, 8, p.hostname, st) - p.buf = encodeField(p.buf, p.bm, 9, p.appVersion, st) - - p.encodeAttributes(p.bm, 10, p.attributes, st) - - p.encodeTraceChunks(p.bm, 11, p.chunks, st) + if p.st == nil { + p.st = newStringTable() + } + p.buf = encodeField(p.buf, p.bm, 2, p.containerID, p.st) + p.buf = encodeField(p.buf, p.bm, 3, p.languageName, p.st) + p.buf = encodeField(p.buf, p.bm, 4, p.languageVersion, p.st) + p.buf = encodeField(p.buf, p.bm, 5, p.tracerVersion, p.st) + p.buf = encodeField(p.buf, p.bm, 6, p.runtimeID, p.st) + p.buf = encodeField(p.buf, p.bm, 7, p.env, p.st) + p.buf = encodeField(p.buf, p.bm, 8, p.hostname, p.st) + p.buf = encodeField(p.buf, p.bm, 9, p.appVersion, p.st) + + p.encodeAttributes(p.bm, 10, p.attributes, p.st) + + p.encodeTraceChunks(p.bm, 11, p.chunks, p.st) } type fieldValue interface { From a4393aeb3707fcaceea7f32ab1684226b98092b1 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 1 Apr 2026 13:19:42 -0400 Subject: [PATCH 12/18] trigger pipelines From aa8ebfa736e26f88717af84ac20ce83477a88d47 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 1 Apr 2026 14:07:10 -0400 Subject: [PATCH 13/18] delete outdated static checks workflow --- .github/workflows/datadog-static-analysis.yml | 25 ------------------- 1 file changed, 25 deletions(-) delete mode 100644 .github/workflows/datadog-static-analysis.yml diff --git a/.github/workflows/datadog-static-analysis.yml b/.github/workflows/datadog-static-analysis.yml deleted file mode 100644 index 93f13cd0171..00000000000 --- a/.github/workflows/datadog-static-analysis.yml +++ /dev/null @@ -1,25 +0,0 @@ -on: [push] - -name: Datadog Static Analysis - -permissions: - contents: read - pull-requests: write - -jobs: - static-analysis: - runs-on: ubuntu-latest - name: Datadog Static Analyzer - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Check code meets quality and security standards - id: datadog-static-analysis - uses: DataDog/datadog-static-analyzer-github-action@8340f18875fcefca86844b5f947ce2431387e552 # v3.0.0 - with: - dd_api_key: ${{ secrets.STATIC_ANALYZER_API_KEY }} - dd_app_key: ${{ secrets.STATIC_ANALYZER_APP_KEY }} - dd_service: dd-trace-go - dd_env: ci - dd_site: datadoghq.com - cpu_count: 2 From 30e36d252f0435a9645d2dc739cb11ea4028b631 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 1 Apr 2026 14:08:02 -0400 Subject: [PATCH 14/18] Revert "delete outdated static checks workflow" This reverts commit aa8ebfa736e26f88717af84ac20ce83477a88d47. --- .github/workflows/datadog-static-analysis.yml | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/datadog-static-analysis.yml diff --git a/.github/workflows/datadog-static-analysis.yml b/.github/workflows/datadog-static-analysis.yml new file mode 100644 index 00000000000..93f13cd0171 --- /dev/null +++ b/.github/workflows/datadog-static-analysis.yml @@ -0,0 +1,25 @@ +on: [push] + +name: Datadog Static Analysis + +permissions: + contents: read + pull-requests: write + +jobs: + static-analysis: + runs-on: ubuntu-latest + name: Datadog Static Analyzer + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Check code meets quality and security standards + id: datadog-static-analysis + uses: DataDog/datadog-static-analyzer-github-action@8340f18875fcefca86844b5f947ce2431387e552 # v3.0.0 + with: + dd_api_key: ${{ secrets.STATIC_ANALYZER_API_KEY }} + dd_app_key: ${{ secrets.STATIC_ANALYZER_APP_KEY }} + dd_service: dd-trace-go + dd_env: ci + dd_site: datadoghq.com + cpu_count: 2 From 1f272f8338b052b6c0e7a824cf942a8195f5d9fd Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Fri, 3 Apr 2026 10:26:08 -0400 Subject: [PATCH 15/18] use built in string type for indices map --- ddtrace/tracer/payload_v1.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index c3edeaa2d0e..f56a5607b1b 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -1039,15 +1039,15 @@ func (s *stringValue) decode(buf []byte) ([]byte, error) { var errUnableDecodeString = errors.New("unable to read string value") type stringTable struct { - strings []stringValue // list of strings - indices map[stringValue]index // map strings to their indices - nextIndex index // last index of the stringTable + strings []stringValue // list of strings + indices map[string]index // map strings to their indices + nextIndex index // last index of the stringTable } func newStringTable() *stringTable { st := &stringTable{ strings: make([]stringValue, 1, 64), - indices: make(map[stringValue]index, 64), + indices: make(map[string]index, 64), nextIndex: 1, } st.strings[0] = "" @@ -1065,12 +1065,15 @@ func (st *stringTable) reset() { // Adds a string to the string table if it does not already exist. func (st *stringTable) serialize(value string, buf []byte) []byte { - sv := stringValue(value) - if idx, ok := st.indices[sv]; ok { + if value == "" { + return msgp.AppendUint32(buf, 0) + } + if idx, ok := st.indices[value]; ok { return idx.encode(buf) } + sv := stringValue(value) buf = sv.encode(buf) - st.indices[sv] = st.nextIndex + st.indices[value] = st.nextIndex st.strings = append(st.strings, sv) st.nextIndex++ return buf From 64548fb0d57278f9007dff73f890d5497f4ca4c5 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 13 Apr 2026 12:29:23 -0400 Subject: [PATCH 16/18] reset string table on encode --- ddtrace/tracer/payload_v1.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index f56a5607b1b..7de8bb04459 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -404,6 +404,8 @@ func (p *payloadV1) update() { func (p *payloadV1) encode() { if p.st == nil { p.st = newStringTable() + } else { + p.st.reset() } p.buf = encodeField(p.buf, p.bm, 2, p.containerID, p.st) p.buf = encodeField(p.buf, p.bm, 3, p.languageName, p.st) From de0fcfc3852f546209b657d6670ec15ec4da3c0f Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 13 Apr 2026 12:46:55 -0400 Subject: [PATCH 17/18] fix: use warnUnsupportedValue func instead of log.Warn --- ddtrace/tracer/payload_v1.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 7de8bb04459..0e535d25c69 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -454,7 +454,7 @@ func encodeField[F fieldValue](buf []byte, bm bitmap, fieldID uint32, a F, st *s buf = v.encode(buf, st) } default: - log.Warn("failed to serialize unsupported fieldValue type for field %d", fieldID) + warnUnsupportedValue(fieldID) buf = st.serialize(serializationFailed, buf) } return buf From 18d426e411887acf6b2d43c6abc1a4bc23940bdf Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 13 Apr 2026 17:20:14 -0400 Subject: [PATCH 18/18] wip: more benchmarking for payloads --- ddtrace/tracer/payload_test.go | 251 +++++++++++++++++++++++++++++++++ 1 file changed, 251 insertions(+) diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index f791c061fe8..704fd21eca5 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -57,6 +57,39 @@ func newDetailedSpanList(n int) spanList { return list } +// creates a list of n spans, populated with repetitive tags +func newLowCardinalitySpanList(n int) spanList { + itoa := map[int]string{0: "0", 1: "1", 2: "2", 3: "3", 4: "4", 5: "5"} + list := make([]*Span, n) + for i := range n { + list[i] = newBasicSpan("span.list." + itoa[i%5+1]) + list[i].start = fixedTime + list[i].service = "high-cardinality-string-value" + list[i].resource = "resource." + itoa[i%5+1] + list[i].SetTag("tag.1", "high-cardinality-string-value") + list[i].SetTag("tag.2", "high-cardinality-string-value") + list[i].SetTag("tag.3", "high-cardinality-string-value") + list[i].SetTag("tag.4", "high-cardinality-string-value") + } + return list +} + +// creates a list of n spans, populated with many unique tags +func newHighCardinalitySpanList(n int) spanList { + itoa := map[int]string{0: "0", 1: "1", 2: "2", 3: "3", 4: "4", 5: "5"} + list := make([]*Span, n) + for i := range n { + list[i] = newBasicSpan("span.list." + itoa[i%5+1]) + list[i].start = fixedTime + list[i].service = "service." + itoa[i%5+1] + list[i].resource = "resource." + itoa[i%5+1] + for i := range 50 { + list[i].SetTag("tag."+itoa[i%5+1], "value."+itoa[i%5+1]) + } + } + return list +} + // TestPayloadIntegrity tests that whatever we push into the payload // allows us to read the same content as would have been encoded by // the codec. @@ -916,3 +949,221 @@ func BenchmarkPayloadVersions(b *testing.B) { }) } } + +func BenchmarkPayloads(b *testing.B) { + b.Run("v0.4", func(b *testing.B) { + b.Run("push/10spans", func(b *testing.B) { + p := newPayloadV04() + sl := newSpanList(10) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + _, _ = p.push(sl) + } + }) + + b.Run("push/1000spans", func(b *testing.B) { + p := newPayloadV04() + sl := newSpanList(1000) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + _, _ = p.push(sl) + } + }) + + b.Run("push/10_detailed_spans", func(b *testing.B) { + p := newPayloadV04() + sl := newDetailedSpanList(10) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + _, _ = p.push(sl) + } + }) + + b.Run("push/1000_detailed_spans", func(b *testing.B) { + p := newPayloadV04() + sl := newDetailedSpanList(1000) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + _, _ = p.push(sl) + } + }) + + b.Run("push/low_cardinality_spans", func(b *testing.B) { + p := newPayloadV04() + sl := newLowCardinalitySpanList(1000) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + _, _ = p.push(sl) + } + }) + + b.Run("push/high_cardinality_spans", func(b *testing.B) { + p := newPayloadV04() + sl := newHighCardinalitySpanList(1000) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + _, _ = p.push(sl) + } + }) + + b.Run("flush/1span", func(b *testing.B) { + p := newPayloadV04() + + p.push(newSpanList(1)) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + p.reset() + io.ReadAll(p) + } + }) + + b.Run("flush/100spans", func(b *testing.B) { + p := newPayloadV04() + + p.push(newSpanList(100)) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + p.reset() + io.ReadAll(p) + } + }) + + b.Run("flush/1000spans", func(b *testing.B) { + p := newPayloadV04() + + p.push(newSpanList(1000)) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + p.reset() + io.ReadAll(p) + } + }) + }) + + b.Run("v1", func(b *testing.B) { + b.Run("push/10spans", func(b *testing.B) { + p := newPayloadV1() + sl := newSpanList(10) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + _, _ = p.push(sl) + } + }) + + b.Run("push/1000spans", func(b *testing.B) { + p := newPayloadV1() + sl := newSpanList(1000) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + _, _ = p.push(sl) + } + }) + + b.Run("push/10_detailed_spans", func(b *testing.B) { + p := newPayloadV1() + sl := newDetailedSpanList(10) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + _, _ = p.push(sl) + } + }) + + b.Run("push/1000_detailed_spans", func(b *testing.B) { + p := newPayloadV1() + sl := newDetailedSpanList(1000) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + _, _ = p.push(sl) + } + }) + + b.Run("push/low_cardinality_spans", func(b *testing.B) { + p := newPayloadV1() + sl := newLowCardinalitySpanList(1000) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + _, _ = p.push(sl) + } + }) + + b.Run("push/high_cardinality_spans", func(b *testing.B) { + p := newPayloadV1() + sl := newHighCardinalitySpanList(1000) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + _, _ = p.push(sl) + } + }) + + b.Run("flush/1span", func(b *testing.B) { + p := newPayloadV1() + + p.push(newSpanList(1)) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + p.reset() + io.ReadAll(p) + } + }) + + b.Run("flush/100spans", func(b *testing.B) { + p := newPayloadV1() + + p.push(newSpanList(100)) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + p.reset() + io.ReadAll(p) + } + }) + + b.Run("flush/1000spans", func(b *testing.B) { + p := newPayloadV1() + + p.push(newSpanList(1000)) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + p.reset() + io.ReadAll(p) + } + }) + }) + + // ... Add more payload versions here... +}