From 8d37bda367ecc039f45fafca08bece8c4547732e Mon Sep 17 00:00:00 2001 From: Fiona Liao Date: Wed, 4 Jun 2025 17:25:30 +0100 Subject: [PATCH 1/8] Allow primitive OTEL delta ingestion to be enabled --- pkg/distributor/otel.go | 6 +- pkg/distributor/otel_test.go | 218 ++++++++++++++++++++++++++++++++++ pkg/distributor/push_test.go | 2 + pkg/util/validation/limits.go | 6 + 4 files changed, 231 insertions(+), 1 deletion(-) diff --git a/pkg/distributor/otel.go b/pkg/distributor/otel.go index a2c3d619f1e..594269db89c 100644 --- a/pkg/distributor/otel.go +++ b/pkg/distributor/otel.go @@ -59,6 +59,7 @@ type OTLPHandlerLimits interface { PromoteOTelResourceAttributes(id string) []string OTelKeepIdentifyingResourceAttributes(id string) bool OTelConvertHistogramsToNHCB(id string) bool + OTelNativeDeltaIngestion(id string) bool } // OTLPHandler is an http.Handler accepting OTLP write requests. @@ -277,11 +278,12 @@ func newOTLPParser( promoteResourceAttributes := resourceAttributePromotionConfig.PromoteOTelResourceAttributes(tenantID) keepIdentifyingResourceAttributes := limits.OTelKeepIdentifyingResourceAttributes(tenantID) convertHistogramsToNHCB := limits.OTelConvertHistogramsToNHCB(tenantID) + allowDeltaTemporality := limits.OTelNativeDeltaIngestion(tenantID) pushMetrics.IncOTLPRequest(tenantID) pushMetrics.ObserveUncompressedBodySize(tenantID, float64(uncompressedBodySize)) - metrics, metricsDropped, err := otelMetricsToTimeseries(ctx, otlpConverter, addSuffixes, enableCTZeroIngestion, enableStartTimeQuietZero, promoteResourceAttributes, keepIdentifyingResourceAttributes, convertHistogramsToNHCB, otlpReq.Metrics(), spanLogger) + metrics, metricsDropped, err := otelMetricsToTimeseries(ctx, otlpConverter, addSuffixes, enableCTZeroIngestion, enableStartTimeQuietZero, promoteResourceAttributes, keepIdentifyingResourceAttributes, convertHistogramsToNHCB, allowDeltaTemporality, otlpReq.Metrics(), spanLogger) if metricsDropped > 0 { discardedDueToOtelParseError.WithLabelValues(tenantID, "").Add(float64(metricsDropped)) // "group" label is empty here as metrics couldn't be parsed } @@ -515,6 +517,7 @@ func otelMetricsToTimeseries( promoteResourceAttributes []string, keepIdentifyingResourceAttributes bool, convertHistogramsToNHCB bool, + allowDeltaTemporality bool, md pmetric.Metrics, logger log.Logger, ) ([]mimirpb.PreallocTimeseries, int, error) { @@ -525,6 +528,7 @@ func otelMetricsToTimeseries( PromoteResourceAttributes: otlp.NewPromoteResourceAttributes(config.OTLPConfig{PromoteResourceAttributes: promoteResourceAttributes}), KeepIdentifyingResourceAttributes: keepIdentifyingResourceAttributes, ConvertHistogramsToNHCB: convertHistogramsToNHCB, + AllowDeltaTemporality: allowDeltaTemporality, } mimirTS := converter.ToTimeseries(ctx, md, settings, logger) diff --git a/pkg/distributor/otel_test.go b/pkg/distributor/otel_test.go index a08c0e4bfe8..46f67a15cea 100644 --- a/pkg/distributor/otel_test.go +++ b/pkg/distributor/otel_test.go @@ -287,6 +287,7 @@ func TestOTelMetricsToTimeSeries(t *testing.T) { tc.promoteResourceAttributes, tc.keepIdentifyingResourceAttributes, false, + false, md, log.NewNopLogger(), ) @@ -362,6 +363,7 @@ func TestConvertOTelHistograms(t *testing.T) { []string{}, false, convertHistogramsToNHCB, + false, md, log.NewNopLogger(), ) @@ -394,6 +396,222 @@ func TestConvertOTelHistograms(t *testing.T) { } } +func TestOTelDeltaIngestion(t *testing.T) { + ts := time.Unix(100, 0) + + testCases := []struct { + name string + allowDelta bool + input pmetric.Metrics + expected prompb.TimeSeries + expectedErr string + }{ + { + name: "delta counter not allowed", + allowDelta: false, + input: func() pmetric.Metrics { + md := pmetric.NewMetrics() + rm := md.ResourceMetrics().AppendEmpty() + il := rm.ScopeMetrics().AppendEmpty() + m := il.Metrics().AppendEmpty() + m.SetName("test_metric") + sum := m.SetEmptySum() + sum.SetAggregationTemporality(pmetric.AggregationTemporalityDelta) + dp := sum.DataPoints().AppendEmpty() + dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) + dp.Attributes().PutStr("metric-attr", "metric value") + return md + }(), + expectedErr: `otlp parse error: invalid temporality and type combination for metric "test_metric"`, + }, + { + name: "delta counter allowed", + allowDelta: true, + input: func() pmetric.Metrics { + md := pmetric.NewMetrics() + rm := md.ResourceMetrics().AppendEmpty() + il := rm.ScopeMetrics().AppendEmpty() + m := il.Metrics().AppendEmpty() + m.SetName("test_metric") + sum := m.SetEmptySum() + sum.SetAggregationTemporality(pmetric.AggregationTemporalityDelta) + dp := sum.DataPoints().AppendEmpty() + dp.SetTimestamp(pcommon.NewTimestampFromTime(ts)) + dp.SetIntValue(5) + dp.Attributes().PutStr("metric-attr", "metric value") + return md + }(), + expected: prompb.TimeSeries{ + Labels: []prompb.Label{{Name: "__name__", Value: "test_metric"}, {Name: "metric-attr", Value: "metric value"}}, + Samples: []prompb.Sample{{Timestamp: ts.UnixMilli(), Value: 5}}, + }, + }, + { + name: "delta exponential histogram not allowed", + allowDelta: false, + input: func() pmetric.Metrics { + md := pmetric.NewMetrics() + rm := md.ResourceMetrics().AppendEmpty() + il := rm.ScopeMetrics().AppendEmpty() + m := il.Metrics().AppendEmpty() + m.SetName("test_metric") + sum := m.SetEmptyExponentialHistogram() + sum.SetAggregationTemporality(pmetric.AggregationTemporalityDelta) + dp := sum.DataPoints().AppendEmpty() + dp.SetCount(1) + dp.SetSum(5) + dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) + dp.Attributes().PutStr("metric-attr", "metric value") + return md + }(), + expectedErr: `otlp parse error: invalid temporality and type combination for metric "test_metric"`, + }, + { + name: "delta exponential histogram allowed", + allowDelta: true, + input: func() pmetric.Metrics { + md := pmetric.NewMetrics() + rm := md.ResourceMetrics().AppendEmpty() + il := rm.ScopeMetrics().AppendEmpty() + m := il.Metrics().AppendEmpty() + m.SetName("test_metric") + sum := m.SetEmptyExponentialHistogram() + sum.SetAggregationTemporality(pmetric.AggregationTemporalityDelta) + dp := sum.DataPoints().AppendEmpty() + dp.SetCount(1) + dp.SetSum(5) + dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) + dp.Attributes().PutStr("metric-attr", "metric value") + return md + }(), + expected: prompb.TimeSeries{ + Labels: []prompb.Label{{Name: "__name__", Value: "test_metric"}, {Name: "metric-attr", Value: "metric value"}}, + Histograms: []prompb.Histogram{ + { + Count: &prompb.Histogram_CountInt{CountInt: 1}, + Sum: 5, + Schema: 0, + ZeroThreshold: 1e-128, + ZeroCount: &prompb.Histogram_ZeroCountInt{ZeroCountInt: 0}, + Timestamp: ts.UnixMilli(), + ResetHint: prompb.Histogram_UNKNOWN, + }, + }, + }, + }, + { + name: "delta histogram as nhcb not allowed", + allowDelta: false, + input: func() pmetric.Metrics { + md := pmetric.NewMetrics() + rm := md.ResourceMetrics().AppendEmpty() + il := rm.ScopeMetrics().AppendEmpty() + m := il.Metrics().AppendEmpty() + m.SetName("test_metric") + sum := m.SetEmptyHistogram() + sum.SetAggregationTemporality(pmetric.AggregationTemporalityDelta) + dp := sum.DataPoints().AppendEmpty() + dp.SetCount(20) + dp.SetSum(30) + dp.BucketCounts().FromRaw([]uint64{10, 10, 0}) + dp.ExplicitBounds().FromRaw([]float64{1, 2}) + dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) + dp.Attributes().PutStr("metric-attr", "metric value") + return md + }(), + expectedErr: `otlp parse error: invalid temporality and type combination for metric "test_metric"`, + }, + { + name: "delta histogram as nhcb allowed", + allowDelta: true, + input: func() pmetric.Metrics { + md := pmetric.NewMetrics() + rm := md.ResourceMetrics().AppendEmpty() + il := rm.ScopeMetrics().AppendEmpty() + m := il.Metrics().AppendEmpty() + m.SetName("test_metric") + sum := m.SetEmptyHistogram() + sum.SetAggregationTemporality(pmetric.AggregationTemporalityDelta) + dp := sum.DataPoints().AppendEmpty() + dp.SetCount(20) + dp.SetSum(30) + dp.BucketCounts().FromRaw([]uint64{10, 10, 0}) + dp.ExplicitBounds().FromRaw([]float64{1, 2}) + dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) + dp.Attributes().PutStr("metric-attr", "metric value") + return md + }(), + expected: prompb.TimeSeries{ + Labels: []prompb.Label{{Name: "__name__", Value: "test_metric"}, {Name: "metric-attr", Value: "metric value"}}, + Histograms: []prompb.Histogram{ + { + Count: &prompb.Histogram_CountInt{CountInt: 20}, + Sum: 30, + Schema: -53, + ZeroThreshold: 0, + ZeroCount: nil, + PositiveSpans: []prompb.BucketSpan{ + { + Length: 3, + }, + }, + PositiveDeltas: []int64{10, 0, -10}, + CustomValues: []float64{1, 2}, + Timestamp: ts.UnixMilli(), + ResetHint: prompb.Histogram_UNKNOWN, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + converter := newOTLPMimirConverter() + mimirTS, dropped, err := otelMetricsToTimeseries( + context.Background(), + converter, + true, + false, + false, + []string{}, + false, + true, + tc.allowDelta, + tc.input, + log.NewNopLogger(), + ) + if tc.expectedErr != "" { + require.EqualError(t, err, tc.expectedErr) + require.Len(t, mimirTS, 0) + require.Equal(t, 1, dropped) + + } else { + require.NoError(t, err) + require.Len(t, mimirTS, 1) + require.Equal(t, 0, dropped) + } + + /*var ts mimirpb.PreallocTimeseries + for i := range mimirTS { + for _, lbl := range mimirTS[i].Labels { + if lbl.Name != labels.MetricName { + continue + } + + if lbl.Value == "target_info" { + continue + } else { + ts = mimirTS[i] + break + } + } + }*/ + + }) + } +} + func BenchmarkOTLPHandler(b *testing.B) { var samples []prompb.Sample for i := 0; i < 1000; i++ { diff --git a/pkg/distributor/push_test.go b/pkg/distributor/push_test.go index 7cf62acad24..1aba7359a7b 100644 --- a/pkg/distributor/push_test.go +++ b/pkg/distributor/push_test.go @@ -1541,6 +1541,8 @@ func (o otlpLimitsMock) OTelKeepIdentifyingResourceAttributes(string) bool { func (o otlpLimitsMock) OTelConvertHistogramsToNHCB(string) bool { return false } +func (o otlpLimitsMock) OTelNativeDeltaIngestion(string) bool { return false } + func promToMimirHistogram(h *prompb.Histogram) mimirpb.Histogram { pSpans := make([]mimirpb.BucketSpan, 0, len(h.PositiveSpans)) for _, span := range h.PositiveSpans { diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 58848772812..0ae96d14997 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -274,6 +274,7 @@ type Limits struct { PromoteOTelResourceAttributes flagext.StringSliceCSV `yaml:"promote_otel_resource_attributes" json:"promote_otel_resource_attributes" category:"experimental"` OTelKeepIdentifyingResourceAttributes bool `yaml:"otel_keep_identifying_resource_attributes" json:"otel_keep_identifying_resource_attributes" category:"experimental"` OTelConvertHistogramsToNHCB bool `yaml:"otel_convert_histograms_to_nhcb" json:"otel_convert_histograms_to_nhcb" category:"experimental"` + OTelNativeDeltaIngestion bool `yaml:"otel_native_delta_ingestion" json:"otel_native_delta_ingestion" category:"experimental"` // Ingest storage. IngestStorageReadConsistency string `yaml:"ingest_storage_read_consistency" json:"ingest_storage_read_consistency" category:"experimental"` @@ -314,6 +315,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.Var(&l.PromoteOTelResourceAttributes, "distributor.otel-promote-resource-attributes", "Optionally specify OTel resource attributes to promote to labels.") f.BoolVar(&l.OTelKeepIdentifyingResourceAttributes, "distributor.otel-keep-identifying-resource-attributes", false, "Whether to keep identifying OTel resource attributes in the target_info metric on top of converting to job and instance labels.") f.BoolVar(&l.OTelConvertHistogramsToNHCB, "distributor.otel-convert-histograms-to-nhcb", false, "Whether to convert OTel explicit histograms into native histograms with custom buckets.") + f.BoolVar(&l.OTelNativeDeltaIngestion, "distributor.otel-native-delta-ingestion", false, "Whether to enable native ingestion of delta OTLP metrics, which will store the raw delta sample values without conversion. If disabled, delta metrics will be rejected. Delta support is in an early stage of development. The ingestion and querying process is likely to change over time.") f.Var(&l.IngestionArtificialDelay, "distributor.ingestion-artificial-delay", "Target ingestion delay to apply to all tenants. If set to a non-zero value, the distributor will artificially delay ingestion time-frame by the specified duration by computing the difference between actual ingestion and the target. There is no delay on actual ingestion of samples, it is only the response back to the client.") f.IntVar(&l.IngestionArtificialDelayConditionForTenantsWithLessThanMaxSeries, "distributor.ingestion-artificial-delay-condition-for-tenants-with-less-than-max-series", 0, "Condition to select tenants for which -distributor.ingestion-artificial-delay-duration-for-tenants-with-less-than-max-series should be applied.") @@ -1257,6 +1259,10 @@ func (o *Overrides) OTelConvertHistogramsToNHCB(tenantID string) bool { return o.getOverridesForUser(tenantID).OTelConvertHistogramsToNHCB } +func (o *Overrides) OTelNativeDeltaIngestion(tenantID string) bool { + return o.getOverridesForUser(tenantID).OTelNativeDeltaIngestion +} + // DistributorIngestionArtificialDelay returns the artificial ingestion latency for a given user. func (o *Overrides) DistributorIngestionArtificialDelay(tenantID string) time.Duration { overrides := o.getOverridesForUser(tenantID) From 26fca3ea239bcdfd8c7f3d76d54565654511cef7 Mon Sep 17 00:00:00 2001 From: Fiona Liao Date: Wed, 4 Jun 2025 17:54:00 +0100 Subject: [PATCH 2/8] Update docs --- cmd/mimir/config-descriptor.json | 11 +++++++++++ cmd/mimir/help-all.txt.tmpl | 2 ++ docs/sources/mimir/configure/about-versioning.md | 2 ++ .../mimir/configure/configuration-parameters/index.md | 7 +++++++ 4 files changed, 22 insertions(+) diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index 416974f99c0..559436ceb25 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -5703,6 +5703,17 @@ "fieldType": "boolean", "fieldCategory": "experimental" }, + { + "kind": "field", + "name": "otel_native_delta_ingestion", + "required": false, + "desc": "Whether to enable native ingestion of delta OTLP metrics, which will store the raw delta sample values without conversion. If disabled, delta metrics will be rejected. Delta support is in an early stage of development. The ingestion and querying process is likely to change over time.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "distributor.otel-native-delta-ingestion", + "fieldType": "boolean", + "fieldCategory": "experimental" + }, { "kind": "field", "name": "ingest_storage_read_consistency", diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index e5d88414a2e..9bba7dcc4cb 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -1425,6 +1425,8 @@ Usage of ./cmd/mimir/mimir: [experimental] Whether to keep identifying OTel resource attributes in the target_info metric on top of converting to job and instance labels. -distributor.otel-metric-suffixes-enabled Whether to enable automatic suffixes to names of metrics ingested through OTLP. + -distributor.otel-native-delta-ingestion + [experimental] Whether to enable native ingestion of delta OTLP metrics, which will store the raw delta sample values without conversion. If disabled, delta metrics will be rejected. Delta support is in an early stage of development. The ingestion and querying process is likely to change over time. -distributor.otel-promote-resource-attributes comma-separated-list-of-strings [experimental] Optionally specify OTel resource attributes to promote to labels. -distributor.remote-timeout duration diff --git a/docs/sources/mimir/configure/about-versioning.md b/docs/sources/mimir/configure/about-versioning.md index 078ffd107cc..2ff38097626 100644 --- a/docs/sources/mimir/configure/about-versioning.md +++ b/docs/sources/mimir/configure/about-versioning.md @@ -112,6 +112,8 @@ The following features are currently experimental: - `-distributor.otel-keep-identifying-resource-attributes` - Enable conversion of OTel explicit bucket histograms into native histograms with custom buckets. - `-distributor.otel-convert-histograms-to-nhcb` + - Enable native ingestion of delta OTLP metrics. Currently, this means storing the raw delta sample values without converting them to cumulative and having the metric type set to "Unknown". Delta support is in an early stage of development. The ingestion and querying process is likely to change over time. Considerations around querying and gotchas can be found in the [corresponding Prometheus documentation](https://prometheus.io/docs/prometheus/3.4/feature_flags/#otlp-native-delta-support). + - `distributor.otel-native-delta-ingestion` - Hash ring - Disabling ring heartbeat timeouts - `-distributor.ring.heartbeat-timeout=0` diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md index 601b7e37f9d..6dd8ac3c748 100644 --- a/docs/sources/mimir/configure/configuration-parameters/index.md +++ b/docs/sources/mimir/configure/configuration-parameters/index.md @@ -4114,6 +4114,13 @@ ruler_alertmanager_client_config: # CLI flag: -distributor.otel-convert-histograms-to-nhcb [otel_convert_histograms_to_nhcb: | default = false] +# (experimental) Whether to enable native ingestion of delta OTLP metrics, which +# will store the raw delta sample values without conversion. If disabled, delta +# metrics will be rejected. Delta support is in an early stage of development. +# The ingestion and querying process is likely to change over time. +# CLI flag: -distributor.otel-native-delta-ingestion +[otel_native_delta_ingestion: | default = false] + # (experimental) The default consistency level to enforce for queries when using # the ingest storage. Supports values: strong, eventual. # CLI flag: -ingest-storage.read-consistency From eea92fb5d7a0d4bce32daaf23c60c76b4adf619f Mon Sep 17 00:00:00 2001 From: Fiona Liao Date: Wed, 4 Jun 2025 18:09:07 +0100 Subject: [PATCH 3/8] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b69b2c6e12f..076834fa4b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ * [FEATURE] Distributor: Experimental support for Prometheus Remote-Write 2.0 protocol. Limitations: Created timestamp is ignored, per series metadata is merged on metric family level automatically, ingestion might fail if client sends ProtoBuf fields out of order. The label `version` is added to the metric `cortex_distributor_requests_in_total` with a value of either `1.0` or `2.0` depending on the detected Remote-Write protocol. #11100 #11101 #11192 #11143 * [FEATURE] Query-frontend: expand `query-frontend.cache-errors` and `query-frontend.results-cache-ttl-for-errors` configuration options to cache non-transient response failures for instant queries. #11120 * [FEATURE] Querier, query-frontend, ruler: Enable experimental support for duration expressions in PromQL, which are simple arithmetics on numbers in offset and range specification. #11344 +* [FEATURE] Distributor: Add experimental `-distributor.otel-native-delta-ingestion` option to allow primitive delta metrics ingestion via the OTLP endpoint. #11631 * [ENHANCEMENT] Dashboards: Add "Added Latency" row to Writes Dashboard. #11579 * [ENHANCEMENT] Ingester: Add support for exporting native histogram cost attribution metrics (`cortex_ingester_attributed_active_native_histogram_series` and `cortex_ingester_attributed_active_native_histogram_buckets`) with labels specified by customers to a custom Prometheus registry. #10892 * [ENHANCEMENT] Store-gateway: Download sparse headers uploaded by compactors. Compactors have to be configured with `-compactor.upload-sparse-index-headers=true` option. #10879 #11072. From 9ec66dea454c6bddd84d54d1c8419698b87eace9 Mon Sep 17 00:00:00 2001 From: Fiona Liao Date: Thu, 5 Jun 2025 11:05:49 +0100 Subject: [PATCH 4/8] Make tests stricter and fix values --- pkg/distributor/otel_test.go | 55 +++++++++++++----------------------- 1 file changed, 19 insertions(+), 36 deletions(-) diff --git a/pkg/distributor/otel_test.go b/pkg/distributor/otel_test.go index 46f67a15cea..671415cb9fc 100644 --- a/pkg/distributor/otel_test.go +++ b/pkg/distributor/otel_test.go @@ -403,7 +403,7 @@ func TestOTelDeltaIngestion(t *testing.T) { name string allowDelta bool input pmetric.Metrics - expected prompb.TimeSeries + expected mimirpb.TimeSeries expectedErr string }{ { @@ -441,9 +441,9 @@ func TestOTelDeltaIngestion(t *testing.T) { dp.Attributes().PutStr("metric-attr", "metric value") return md }(), - expected: prompb.TimeSeries{ - Labels: []prompb.Label{{Name: "__name__", Value: "test_metric"}, {Name: "metric-attr", Value: "metric value"}}, - Samples: []prompb.Sample{{Timestamp: ts.UnixMilli(), Value: 5}}, + expected: mimirpb.TimeSeries{ + Labels: []mimirpb.LabelAdapter{{Name: "__name__", Value: "test_metric"}, {Name: "metric_attr", Value: "metric value"}}, + Samples: []mimirpb.Sample{{TimestampMs: ts.UnixMilli(), Value: 5}}, }, }, { @@ -480,21 +480,21 @@ func TestOTelDeltaIngestion(t *testing.T) { dp := sum.DataPoints().AppendEmpty() dp.SetCount(1) dp.SetSum(5) - dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) + dp.SetTimestamp(pcommon.NewTimestampFromTime(ts)) dp.Attributes().PutStr("metric-attr", "metric value") return md }(), - expected: prompb.TimeSeries{ - Labels: []prompb.Label{{Name: "__name__", Value: "test_metric"}, {Name: "metric-attr", Value: "metric value"}}, - Histograms: []prompb.Histogram{ + expected: mimirpb.TimeSeries{ + Labels: []mimirpb.LabelAdapter{{Name: "__name__", Value: "test_metric"}, {Name: "metric_attr", Value: "metric value"}}, + Histograms: []mimirpb.Histogram{ { - Count: &prompb.Histogram_CountInt{CountInt: 1}, + Count: &mimirpb.Histogram_CountInt{CountInt: 1}, Sum: 5, Schema: 0, ZeroThreshold: 1e-128, - ZeroCount: &prompb.Histogram_ZeroCountInt{ZeroCountInt: 0}, + ZeroCount: &mimirpb.Histogram_ZeroCountInt{ZeroCountInt: 0}, Timestamp: ts.UnixMilli(), - ResetHint: prompb.Histogram_UNKNOWN, + ResetHint: mimirpb.Histogram_GAUGE, }, }, }, @@ -537,20 +537,20 @@ func TestOTelDeltaIngestion(t *testing.T) { dp.SetSum(30) dp.BucketCounts().FromRaw([]uint64{10, 10, 0}) dp.ExplicitBounds().FromRaw([]float64{1, 2}) - dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) + dp.SetTimestamp(pcommon.NewTimestampFromTime(ts)) dp.Attributes().PutStr("metric-attr", "metric value") return md }(), - expected: prompb.TimeSeries{ - Labels: []prompb.Label{{Name: "__name__", Value: "test_metric"}, {Name: "metric-attr", Value: "metric value"}}, - Histograms: []prompb.Histogram{ + expected: mimirpb.TimeSeries{ + Labels: []mimirpb.LabelAdapter{{Name: "__name__", Value: "test_metric"}, {Name: "metric_attr", Value: "metric value"}}, + Histograms: []mimirpb.Histogram{ { - Count: &prompb.Histogram_CountInt{CountInt: 20}, + Count: &mimirpb.Histogram_CountInt{CountInt: 20}, Sum: 30, Schema: -53, ZeroThreshold: 0, ZeroCount: nil, - PositiveSpans: []prompb.BucketSpan{ + PositiveSpans: []mimirpb.BucketSpan{ { Length: 3, }, @@ -558,7 +558,7 @@ func TestOTelDeltaIngestion(t *testing.T) { PositiveDeltas: []int64{10, 0, -10}, CustomValues: []float64{1, 2}, Timestamp: ts.UnixMilli(), - ResetHint: prompb.Histogram_UNKNOWN, + ResetHint: mimirpb.Histogram_GAUGE, }, }, }, @@ -585,29 +585,12 @@ func TestOTelDeltaIngestion(t *testing.T) { require.EqualError(t, err, tc.expectedErr) require.Len(t, mimirTS, 0) require.Equal(t, 1, dropped) - } else { require.NoError(t, err) require.Len(t, mimirTS, 1) require.Equal(t, 0, dropped) + require.Equal(t, tc.expected, *mimirTS[0].TimeSeries) } - - /*var ts mimirpb.PreallocTimeseries - for i := range mimirTS { - for _, lbl := range mimirTS[i].Labels { - if lbl.Name != labels.MetricName { - continue - } - - if lbl.Value == "target_info" { - continue - } else { - ts = mimirTS[i] - break - } - } - }*/ - }) } } From 6a1119d8c9df3772211d26b8103ca0d62346ba98 Mon Sep 17 00:00:00 2001 From: Fiona Liao Date: Fri, 4 Jul 2025 15:08:46 +0100 Subject: [PATCH 5/8] Update docs/sources/mimir/configure/about-versioning.md Co-authored-by: Taylor C <41653732+tacole02@users.noreply.github.com> --- docs/sources/mimir/configure/about-versioning.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/mimir/configure/about-versioning.md b/docs/sources/mimir/configure/about-versioning.md index ac66bacc742..fae08578766 100644 --- a/docs/sources/mimir/configure/about-versioning.md +++ b/docs/sources/mimir/configure/about-versioning.md @@ -113,7 +113,7 @@ The following features are currently experimental: - `-distributor.otel-keep-identifying-resource-attributes` - Enable conversion of OTel explicit bucket histograms into native histograms with custom buckets. - `-distributor.otel-convert-histograms-to-nhcb` - - Enable native ingestion of delta OTLP metrics. Currently, this means storing the raw delta sample values without converting them to cumulative and having the metric type set to "Unknown". Delta support is in an early stage of development. The ingestion and querying process is likely to change over time. Considerations around querying and gotchas can be found in the [corresponding Prometheus documentation](https://prometheus.io/docs/prometheus/3.4/feature_flags/#otlp-native-delta-support). + - Enable native ingestion of delta OTLP metrics. This means storing the raw delta sample values without converting them to cumulative and having the metric type set to "Unknown". Delta support is in an early stage of development. The ingestion and querying process is likely to change over time. You can find considerations around querying and gotchas in the [corresponding Prometheus documentation](https://prometheus.io/docs/prometheus/3.4/feature_flags/#otlp-native-delta-support). - `distributor.otel-native-delta-ingestion` - Hash ring - Disabling ring heartbeat timeouts From 35dfa9157c3d9f87ca98566a681333852a308d80 Mon Sep 17 00:00:00 2001 From: Fiona Liao Date: Fri, 4 Jul 2025 15:08:55 +0100 Subject: [PATCH 6/8] Update docs/sources/mimir/configure/configuration-parameters/index.md Co-authored-by: Taylor C <41653732+tacole02@users.noreply.github.com> --- docs/sources/mimir/configure/configuration-parameters/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md index 2235060c33c..3d1c2133b19 100644 --- a/docs/sources/mimir/configure/configuration-parameters/index.md +++ b/docs/sources/mimir/configure/configuration-parameters/index.md @@ -4148,7 +4148,7 @@ ruler_alertmanager_client_config: [otel_convert_histograms_to_nhcb: | default = false] # (experimental) Whether to enable native ingestion of delta OTLP metrics, which -# will store the raw delta sample values without conversion. If disabled, delta +# stores the raw delta sample values without conversion. If disabled, delta # metrics will be rejected. Delta support is in an early stage of development. # The ingestion and querying process is likely to change over time. # CLI flag: -distributor.otel-native-delta-ingestion From 2360504b12fbabdd6ffc4606702640f45867fb97 Mon Sep 17 00:00:00 2001 From: Fiona Liao Date: Fri, 4 Jul 2025 15:09:03 +0100 Subject: [PATCH 7/8] Update docs/sources/mimir/configure/configuration-parameters/index.md Co-authored-by: Taylor C <41653732+tacole02@users.noreply.github.com> --- docs/sources/mimir/configure/configuration-parameters/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md index 3d1c2133b19..c178742317e 100644 --- a/docs/sources/mimir/configure/configuration-parameters/index.md +++ b/docs/sources/mimir/configure/configuration-parameters/index.md @@ -4149,7 +4149,7 @@ ruler_alertmanager_client_config: # (experimental) Whether to enable native ingestion of delta OTLP metrics, which # stores the raw delta sample values without conversion. If disabled, delta -# metrics will be rejected. Delta support is in an early stage of development. +# metrics are rejected. Delta support is in an early stage of development. # The ingestion and querying process is likely to change over time. # CLI flag: -distributor.otel-native-delta-ingestion [otel_native_delta_ingestion: | default = false] From 42087f5ba4d6a6a7485afb370dd9edcf79234d48 Mon Sep 17 00:00:00 2001 From: Fiona Liao Date: Fri, 4 Jul 2025 15:10:24 +0100 Subject: [PATCH 8/8] Doc update suggestion --- docs/sources/mimir/configure/about-versioning.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/mimir/configure/about-versioning.md b/docs/sources/mimir/configure/about-versioning.md index fae08578766..ee53cbdcf02 100644 --- a/docs/sources/mimir/configure/about-versioning.md +++ b/docs/sources/mimir/configure/about-versioning.md @@ -113,7 +113,7 @@ The following features are currently experimental: - `-distributor.otel-keep-identifying-resource-attributes` - Enable conversion of OTel explicit bucket histograms into native histograms with custom buckets. - `-distributor.otel-convert-histograms-to-nhcb` - - Enable native ingestion of delta OTLP metrics. This means storing the raw delta sample values without converting them to cumulative and having the metric type set to "Unknown". Delta support is in an early stage of development. The ingestion and querying process is likely to change over time. You can find considerations around querying and gotchas in the [corresponding Prometheus documentation](https://prometheus.io/docs/prometheus/3.4/feature_flags/#otlp-native-delta-support). + - Enable native ingestion of delta OTLP metrics. This means storing the raw delta sample values without converting them to cumulative values and having the metric type set to "Unknown". Delta support is in an early stage of development. The ingestion and querying process is likely to change over time. You can find considerations around querying and gotchas in the [corresponding Prometheus documentation](https://prometheus.io/docs/prometheus/3.4/feature_flags/#otlp-native-delta-support). - `distributor.otel-native-delta-ingestion` - Hash ring - Disabling ring heartbeat timeouts