From 3f84e8b4a9b68263368d43bd81607e9fe961b1d5 Mon Sep 17 00:00:00 2001 From: Wei Li Date: Tue, 9 Jun 2026 17:53:13 -0700 Subject: [PATCH 1/4] Add async retry logic to GRPCWriter for OTel Collector outages When the OTel Collector is temporarily unavailable, GRPCWriter previously dropped export batches silently. This adds a background retry mechanism that queues failed batches and retries them with exponential backoff, preventing data loss during collector restarts of up to ~91 seconds. Design: - Retry is asynchronous: withRetry enqueues a failed batch to a per-signal buffered channel and returns immediately, releasing the SignalBatcher mutex so the DiodeWriter ring buffer continues to drain without stalling. - Three independent retry workers (metrics, logs, traces) each maintain a pool of pending batches and replay the entire pool on each backoff tick, flushing the full backlog in one sweep once the collector recovers. - Backoff sequence: 1s,2s,4s,8s,16s,30s,30s (7 attempts, ~91s total). - Per-signal channel capacity 1024, sized to absorb the full retry window (91s*10 batches/s = 910 batches) without overflow. --- src/go.sum | 110 +++++++++ src/pkg/otelcolclient/otelcolclient.go | 230 +++++++++++++++--- src/pkg/otelcolclient/otelcolclient_test.go | 253 ++++++++++++++++++-- 3 files changed, 542 insertions(+), 51 deletions(-) diff --git a/src/go.sum b/src/go.sum index 757a383bf..9e39a02f1 100644 --- a/src/go.sum +++ b/src/go.sum @@ -1,3 +1,11 @@ +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= +cloud.google.com/go/kms v1.31.0/go.mod h1:YIyXZym11R5uovJJt4oN5eUL3oPmirF3yKeIh6QAf4U= +cloud.google.com/go/longrunning v0.9.0/go.mod h1:pkTz846W7bF4o2SzdWJ40Hu0Re+UoNT6Q5t+igIcb8E= code.cloudfoundry.org/go-batching v0.0.0-20260526123032-013946b96a09 h1:0UJG1LYBpGC7rrVjFvmSkfuOYVqiI0DUmLJOktGiU3c= code.cloudfoundry.org/go-batching v0.0.0-20260526123032-013946b96a09/go.mod h1:PoY0RDyVol/cGj+v9DMBznRcG5FgcWD91Uy2iqN/Caw= code.cloudfoundry.org/go-diodes v0.0.0-20260526122959-0284fcb5ac88 h1:9HNd8DjiaHrHwiblgczOwfufgVBlDozkgVv5FxqZcts= @@ -10,22 +18,62 @@ code.cloudfoundry.org/go-metric-registry v0.0.0-20260602223408-eae7443fd7fa h1:v code.cloudfoundry.org/go-metric-registry v0.0.0-20260602223408-eae7443fd7fa/go.mod h1:AKZJZ2b22vZTfc20eev4vao3Z5HUeBCRy5pSAJrijtY= code.cloudfoundry.org/tlsconfig v0.58.0 h1:UGMvB95Q1hkBVEHEAg+8i6pFBmLPRo5XaD+xtJ4E6DQ= code.cloudfoundry.org/tlsconfig v0.58.0/go.mod h1:C7+gD2NiT/00LliHJOGJhpjO9myczswc3uvv2++payw= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1/go.mod h1:pzBXCYn05zvYIrwLgtK8Ap8QcjRg+0i76tMQdWN6wOk= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0/go.mod h1:i2h9fsTFKZorh8RdV2IcSUf/Qj98GlTkrTvUbX/s8as= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= +github.com/ThalesGroup/crypto11 v1.6.0/go.mod h1:H6LRjN5R5SHxTrLqGNteisLDI0/IC6+SGx1pHtbwizE= +github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= +github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apoydence/eachers v0.0.0-20181020210610-23942921fe77/go.mod h1:bXvGk6IkT1Agy7qzJ+DjIw/SJ1AaB3AvAuMDVV+Vkoo= +github.com/aws/aws-sdk-go-v2 v1.41.11/go.mod h1:iiUX27gOXRuYaoeUVXhUpPwjJHzISfPAjjcuhUbLSVs= +github.com/aws/aws-sdk-go-v2/config v1.32.22/go.mod h1:0+H+0nPKbvWltf5vSIGkApv+hGbaQ4FfwTjGIYQREcw= +github.com/aws/aws-sdk-go-v2/credentials v1.19.21/go.mod h1:UE8+9t5zudFwu5k5ShC1PKArVEdOkQQdCXIHQAVNUcU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.27/go.mod h1:ISGSFNbOHRS+JV/17yStzRTPBUHHqF92kCpRLLyH3Nk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.27/go.mod h1:QV9IVIopJ1dpQUno0f9VYDUwOEjj8u0iEJ4JiZVre3Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.27/go.mod h1:x0rldpsnUQaQIs4Rh+Vwm9Z/0vI6BxadGtsgJfZFb8s= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.28/go.mod h1:oTdbDr+BMs7gAYrNpD0LDTyqQfv6yOYgTDv46+xbwFY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.11/go.mod h1:XG68qW+YLLFH0vnSDCou43Cgj5TeAG83O5NRSJgt04Y= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.27/go.mod h1:p7hwgbwompjCRNTdB3ytlldddNt1rDBgVVMqWEVG1II= +github.com/aws/aws-sdk-go-v2/service/kms v1.53.2/go.mod h1:ZNloshpT3zBMbDUwsbbwYP46ADvoJU9/9WGX1WYegwg= +github.com/aws/aws-sdk-go-v2/service/signin v1.1.3/go.mod h1:WhO1EH3phjFWValQDsExaxncgEWJsHeoTvuyQAj3jwU= +github.com/aws/aws-sdk-go-v2/service/sso v1.31.1/go.mod h1:tEL1hqCrkgwrDVL04HuLxz1SLUXdh+4kKhWv1pXKeiY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.4/go.mod h1:0x10Wy0dVS4Gn552xhHY5th2QdYpfJf44EsfyYGV194= +github.com/aws/aws-sdk-go-v2/service/sts v1.43.1/go.mod h1:t01JURC8Fe5M+7R1K0vzIZ2NT04HqvZR+FjlHrHDT2A= +github.com/aws/smithy-go v1.27.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/cloudfoundry/dropsonde v1.1.0 h1:nerhj8K0heOsv/U/Ddou5Esw56YlNeHHJH6MP9QlACQ= github.com/cloudfoundry/dropsonde v1.1.0/go.mod h1:OrkxsBrAvM8X0Ve9vaSNKLR+/Jeohu3+J0M4JEaTmnM= +github.com/cloudfoundry/gosteno v0.0.0-20150423193413-0c8581caea35/go.mod h1:3YBPUR85RIrvaUTdA1dL38YSp6s3OHu1xrWLkGt2Mog= github.com/cloudfoundry/sonde-go v0.0.0-20260526083715-66f310f13c26 h1:DHWZw6RGiApykMdpy4d1t0OApzautzvnJI60H2B5OSM= github.com/cloudfoundry/sonde-go v0.0.0-20260526083715-66f310f13c26/go.mod h1:rR3cyMIEfcaknKHjUDMf7Cuq5gn9Aiqvzz/gqsnsQBI= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= @@ -34,26 +82,47 @@ github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01 github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM= github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= +github.com/go-jose/go-jose/v3 v3.0.5/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-piv/piv-go/v2 v2.6.0/go.mod h1:gcsS2WGbToZk5PwtsCIlCWzV/R+r/wN+Xi+6O6IlU3c= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/certificate-transparency-go v1.1.2/go.mod h1:3OL+HKDqHPUfdKrHVQxO6T8nDLO0HF7LRTlkIWXaWvQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/go-tpm-tools v0.4.8/go.mod h1:4DfiOtiS1KppJjwf1+tqtW4K3PrCJjAAqFKj/TYTJKg= +github.com/google/go-tspi v0.3.0/go.mod h1:xfMGI3G0PhxCdNVcYr1C4C+EizojDg/TXuX5by8CiHI= github.com/google/pprof v0.0.0-20260604005048-7023385849c0 h1:h1QTMDl6q9wDvDCJVpKQSjgleGFYnd2fOxmg2K+6BGE= github.com/google/pprof v0.0.0-20260604005048-7023385849c0/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.16/go.mod h1:9Yb0eAkH/Xqhvv3zbeKf/+wMJqCeocWc6KIhDvEAuYE= +github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= +github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/ianlancetaylor/demangle v0.0.0-20250417193237-f615e6bd150b/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -62,24 +131,38 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2 h1:yVCLo4+ACVroOEr4iFU1iH46Ldlzz2rTuu18Ra7M8sU= github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2/go.mod h1:VzB2VoMh1Y32/QqDfg9ZJYHj99oM4LiGtqPZydTiQSQ= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.29.0 h1:rfh+ZFjgJhYWRoIqVf3Uwx/W20yLrcrE2h2GmYVRaag= github.com/onsi/ginkgo/v2 v2.29.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.41.0 h1:OwKp4pXNgVxf6sCplzYo794OFNuoL2q2SBMU5NSWOjA= github.com/onsi/gomega v1.41.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= +github.com/peterbourgon/diskv/v3 v3.0.1/go.mod h1:kJ5Ny7vLdARGU3WUuy6uzO6T0nb/2gWcT1JiBvRmb5o= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/poy/eachers v0.0.0-20181020210610-23942921fe77/go.mod h1:x1vqpbcMW9T/KRcQ4b48diSiSVtYgvwQ5xzDByEg4WE= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -88,14 +171,24 @@ github.com/prometheus/common v0.68.1 h1:omjRRl4QP4komogpXuhfeOiisQg7xdy8VM1UY+pS github.com/prometheus/common v0.68.1/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/schollz/jsonstore v1.1.0/go.mod h1:15c6+9guw8vDRyozGjN3FoILt0wpruJk9Pi66vjaZfg= github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/smallstep/go-attestation v0.4.4-0.20260603212853-e1a87a0b07d9/go.mod h1:vNAduivU014fubg6ewygkAvQC0IQVXqdc8vaGl/0er4= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/square/certstrap v1.3.0 h1:N9P0ZRA+DjT8pq5fGDj0z3FjafRKnBDypP0QHpMlaAk= github.com/square/certstrap v1.3.0/go.mod h1:wGZo9eE1B7WX2GKBn0htJ+B3OuRl2UsdCFySNooy9hU= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/thales-e-security/pool v0.0.2/go.mod h1:qtpMm2+thHtqhLzTwgDBj/OuNnMpupY8mv0Phz0gjhU= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -104,14 +197,20 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/urfave/cli v1.22.9/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.71.0 h1:tepR7H+Guh9VUqxxcPggYi8R3lGUu2Rsdh+z7/FCY3k= github.com/valyala/fasthttp v1.71.0/go.mod h1:z1sDUvOShhXq/C9mwH/fSm1Vb71tUJwmQdgkBrBNwnA= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= @@ -128,6 +227,7 @@ go.step.sm/crypto v0.82.0 h1:JOT8b/7Jh4My3mxE4U7UkuaN2sUGkZ8fnjznXaTGoRE= go.step.sm/crypto v0.82.0/go.mod h1:qyLTv666WJ6ImFPUjljux+684Y/GGYUjAZcKCnc6yBs= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= @@ -138,18 +238,23 @@ golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6/go.mod h1:Eqhaxk/wZsWEH8CRxLwj6xzEJbz7k1EFGqx7nyCoabE= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.283.0/go.mod h1:6Wssta4c5n9qHq5CBhmlai5h/PUa1djdDAIhYEHyvcM= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= @@ -165,3 +270,8 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= +modernc.org/libc v1.72.3/go.mod h1:dn0dZNnnn1clLyvRxLxYExxiKRZIRENOfqQ8XEeg4Qs= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.51.0/go.mod h1:tcNzv5p84E0skkmJn038y+hWJbLQXQqEnQfeh5r2JLM= diff --git a/src/pkg/otelcolclient/otelcolclient.go b/src/pkg/otelcolclient/otelcolclient.go index ba2de7365..b1afbe7a1 100644 --- a/src/pkg/otelcolclient/otelcolclient.go +++ b/src/pkg/otelcolclient/otelcolclient.go @@ -20,9 +20,17 @@ import ( metricspb "go.opentelemetry.io/proto/otlp/metrics/v1" tracepb "go.opentelemetry.io/proto/otlp/trace/v1" "google.golang.org/grpc" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" ) +// retryItem holds a failed export closure and a count of retry attempts already made. +type retryItem struct { + exportFn func() error + attempts int +} + type GRPCWriter struct { // The client API for the OTel Collector metrics service msc colmetricspb.MetricsServiceClient @@ -41,6 +49,21 @@ type GRPCWriter struct { // The logger to use for errors l *log.Logger + + // Retry configuration. + maxRetries int + initialRetryDelay time.Duration + maxRetryDelay time.Duration + + // Per-signal queues for async retry workers. Items are enqueued by withRetry + // and consumed by runRetryWorker goroutines started in NewGRPCWriter. + // Sized to absorb the full retry window: 7 attempts over 91 s (1+2+4+8+16+30+30) + // at the 100 ms flush interval produces at most 910 batches. 1024 clears that + // with a small margin. A nil channel disables async retry (used in tests that + // construct GRPCWriter directly without starting workers). + metricsRetry chan retryItem + logsRetry chan retryItem + tracesRetry chan retryItem } // NewGRPCWriter dials the provided gRPC address and returns a *GRPCWriter. @@ -58,58 +81,199 @@ func NewGRPCWriter(addr string, tlsConfig *tls.Config, l *log.Logger) (*GRPCWrit }() w := &GRPCWriter{ - msc: colmetricspb.NewMetricsServiceClient(cc), - tsc: coltracepb.NewTraceServiceClient(cc), - lsc: collogspb.NewLogsServiceClient(cc), - ctx: ctx, - cancel: cancel, - l: l, + msc: colmetricspb.NewMetricsServiceClient(cc), + tsc: coltracepb.NewTraceServiceClient(cc), + lsc: collogspb.NewLogsServiceClient(cc), + ctx: ctx, + cancel: cancel, + l: l, + maxRetries: 7, + initialRetryDelay: 1 * time.Second, + maxRetryDelay: 30 * time.Second, + metricsRetry: make(chan retryItem, 1024), + logsRetry: make(chan retryItem, 1024), + tracesRetry: make(chan retryItem, 1024), } + go w.runRetryWorker(w.metricsRetry) + go w.runRetryWorker(w.logsRetry) + go w.runRetryWorker(w.tracesRetry) cancel = nil return w, nil } func (w GRPCWriter) WriteLogs(batch []*logspb.ResourceLogs) { - resp, err := w.lsc.Export(w.ctx, &collogspb.ExportLogsServiceRequest{ - ResourceLogs: batch, - }) - if err == nil { - err = errorOnLogsRejection(resp) - } - if err != nil { - w.l.Println("Write error:", err) - } + w.withRetry(func() error { + resp, err := w.lsc.Export(w.ctx, &collogspb.ExportLogsServiceRequest{ + ResourceLogs: batch, + }) + if err != nil { + return err + } + return errorOnLogsRejection(resp) + }, w.logsRetry) } func (w GRPCWriter) WriteMetrics(batch []*metricspb.Metric) { - resp, err := w.msc.Export(w.ctx, &colmetricspb.ExportMetricsServiceRequest{ - ResourceMetrics: []*metricspb.ResourceMetrics{ - { - ScopeMetrics: []*metricspb.ScopeMetrics{ - { - Metrics: batch, + w.withRetry(func() error { + resp, err := w.msc.Export(w.ctx, &colmetricspb.ExportMetricsServiceRequest{ + ResourceMetrics: []*metricspb.ResourceMetrics{ + { + ScopeMetrics: []*metricspb.ScopeMetrics{ + { + Metrics: batch, + }, }, }, }, - }, - }) - if err == nil { - err = errorOnRejection(resp) + }) + if err != nil { + return err + } + return errorOnRejection(resp) + }, w.metricsRetry) +} + +func (w GRPCWriter) WriteTrace(batch []*tracepb.ResourceSpans) { + w.withRetry(func() error { + resp, err := w.tsc.Export(w.ctx, &coltracepb.ExportTraceServiceRequest{ + ResourceSpans: batch, + }) + if err != nil { + return err + } + return errorOnTraceRejection(resp) + }, w.tracesRetry) +} + +// isRetryable reports whether a gRPC error is transient and worth retrying. +// Only codes.Unavailable is retried — the expected code when the OTel +// Collector is down or restarting. Partial-success rejections (plain errors +// from errorOn*Rejection) return false. +func isRetryable(err error) bool { + s, ok := status.FromError(err) + if !ok { + return false } - if err != nil { - w.l.Println("Write error:", err) + return s.Code() == codes.Unavailable +} + +// isContextError reports whether an error is due to context cancellation so +// that shutdown paths can be distinguished from genuine write failures. +func isContextError(err error) bool { + if errors.Is(err, context.Canceled) { + return true } + s, ok := status.FromError(err) + return ok && s.Code() == codes.Canceled } -func (w GRPCWriter) WriteTrace(batch []*tracepb.ResourceSpans) { - resp, err := w.tsc.Export(w.ctx, &coltracepb.ExportTraceServiceRequest{ - ResourceSpans: batch, - }) +// withRetry makes a single export attempt. If the error is retryable the batch +// is handed off to the background retry worker via queue so the caller (running +// inside the SignalBatcher flush path, holding its mutex) is not blocked during +// backoff. Non-retryable errors and context errors are handled inline. +func (w GRPCWriter) withRetry(exportFn func() error, queue chan<- retryItem) { + err := exportFn() if err == nil { - err = errorOnTraceRejection(resp) + return } - if err != nil { + if isContextError(err) { + return + } + if !isRetryable(err) { w.l.Println("Write error:", err) + return + } + select { + case queue <- retryItem{exportFn: exportFn}: + default: + w.l.Println("Write error (retry queue full):", err) + } +} + +// drainRetryQueue non-blockingly moves all pending items from src into dst. +func drainRetryQueue(dst []retryItem, src <-chan retryItem) []retryItem { + for { + select { + case item := <-src: + dst = append(dst, item) + default: + return dst + } + } +} + +// runRetryWorker retries batches that were queued by withRetry. It maintains a +// pool of pending batches and replays them all on each backoff tick so that +// when the collector recovers the entire backlog is flushed in one sweep rather +// than one batch per backoff cycle. +// +// The shared pool delay resets to initialRetryDelay each time the pool drains +// to empty. Items that exhaust maxRetries are logged and discarded. The +// goroutine exits silently when the writer's context is cancelled. +func (w *GRPCWriter) runRetryWorker(queue <-chan retryItem) { + var pool []retryItem + delay := w.initialRetryDelay + + for { + // Merge any newly queued items into the pool. + prevLen := len(pool) + pool = drainRetryQueue(pool, queue) + if prevLen == 0 && len(pool) > 0 { + delay = w.initialRetryDelay + w.l.Println("New item added to empty pool, delay set to", delay) + } + + if len(pool) == 0 { + // Block until there is work to do or the context is cancelled. + select { + case <-w.ctx.Done(): + return + case item := <-queue: + pool = append(pool, item) + delay = w.initialRetryDelay + w.l.Println("New item added to empty pool, delay set to", delay) + } + pool = drainRetryQueue(pool, queue) + } + + // Wait before the retry attempt. + select { + case <-w.ctx.Done(): + return + case <-time.After(delay): + } + + // Drain items that arrived during the sleep. + pool = drainRetryQueue(pool, queue) + + // Attempt every item in the pool; keep the ones that still need more retries. + var remaining []retryItem + for _, item := range pool { + if isContextError(w.ctx.Err()) { + return + } + err := item.exportFn() + if err == nil { + continue + } + if isContextError(err) { + return + } + item.attempts++ + if !isRetryable(err) || item.attempts >= w.maxRetries { + w.l.Println("Write error:", err) + continue + } + remaining = append(remaining, item) + } + pool = remaining + + if len(pool) == 0 { + delay = w.initialRetryDelay + w.l.Println("Pool is empty, delay set to", delay) + } else { + delay = min(delay*2, w.maxRetryDelay) + } } } diff --git a/src/pkg/otelcolclient/otelcolclient_test.go b/src/pkg/otelcolclient/otelcolclient_test.go index 6aa265524..0ae14a3e7 100644 --- a/src/pkg/otelcolclient/otelcolclient_test.go +++ b/src/pkg/otelcolclient/otelcolclient_test.go @@ -5,6 +5,7 @@ import ( "errors" "log" "math" + "sync" "time" "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" @@ -20,6 +21,8 @@ import ( metricspb "go.opentelemetry.io/proto/otlp/metrics/v1" tracepb "go.opentelemetry.io/proto/otlp/trace/v1" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/testing/protocmp" ) @@ -1056,6 +1059,169 @@ var _ = Describe("Client", func() { }) }) + Describe("retry behavior", func() { + var ( + retryMSC *spyMetricsServiceClient + retryLSC *spyLogsServiceClient + retryTSC *spyTraceServiceClient + retryW *GRPCWriter + retryCancel context.CancelFunc + ) + + BeforeEach(func() { + retryMSC = &spyMetricsServiceClient{ + requests: make(chan *colmetricspb.ExportMetricsServiceRequest, 10), + response: &colmetricspb.ExportMetricsServiceResponse{}, + } + retryLSC = &spyLogsServiceClient{ + requests: make(chan *collogspb.ExportLogsServiceRequest, 10), + response: &collogspb.ExportLogsServiceResponse{}, + } + retryTSC = &spyTraceServiceClient{ + requests: make(chan *coltracepb.ExportTraceServiceRequest, 10), + response: &coltracepb.ExportTraceServiceResponse{}, + } + ctx, cancel := context.WithCancel(context.Background()) + retryCancel = cancel + retryW = &GRPCWriter{ + msc: retryMSC, + tsc: retryTSC, + lsc: retryLSC, + ctx: ctx, + cancel: cancel, + l: log.New(GinkgoWriter, "", 0), + maxRetries: 3, + initialRetryDelay: time.Millisecond, + maxRetryDelay: 5 * time.Millisecond, + metricsRetry: make(chan retryItem, 16), + logsRetry: make(chan retryItem, 16), + tracesRetry: make(chan retryItem, 16), + } + go retryW.runRetryWorker(retryW.metricsRetry) + go retryW.runRetryWorker(retryW.logsRetry) + go retryW.runRetryWorker(retryW.tracesRetry) + }) + + AfterEach(func() { + retryCancel() + }) + + Context("when a metrics export fails with a retryable error then succeeds on retry", func() { + BeforeEach(func() { + retryMSC.responseErrs = []error{ + status.Error(codes.Unavailable, "collector temporarily unavailable"), + nil, + } + }) + + It("retries in the background and does not log a write error", func() { + retryW.WriteMetrics([]*metricspb.Metric{{Name: "test-metric"}}) + + Eventually(func() int { + retryMSC.mu.Lock() + defer retryMSC.mu.Unlock() + return retryMSC.exportCount + }).Should(Equal(2)) + Expect(buf).NotTo(gbytes.Say("Write error")) + }) + }) + + Context("when all retries are exhausted", func() { + BeforeEach(func() { + retryMSC.responseErr = status.Error(codes.Unavailable, "collector down") + }) + + It("logs a write error after the final retry attempt", func() { + retryW.WriteMetrics([]*metricspb.Metric{{Name: "test-metric"}}) + Eventually(buf).Should(gbytes.Say("Write error:.*collector down")) + }) + }) + + Context("when the error is not retryable", func() { + BeforeEach(func() { + retryMSC.responseErr = status.Error(codes.InvalidArgument, "bad metric") + }) + + It("logs the error immediately without queuing a retry", func() { + retryW.WriteMetrics([]*metricspb.Metric{{Name: "test-metric"}}) + + retryMSC.mu.Lock() + count := retryMSC.exportCount + retryMSC.mu.Unlock() + + Expect(count).To(Equal(1)) + Expect(buf).To(gbytes.Say("Write error:.*bad metric")) + }) + }) + + Context("when a logs export fails with ResourceExhausted", func() { + BeforeEach(func() { + retryLSC.responseErr = status.Error(codes.ResourceExhausted, "rate limited") + }) + + It("logs the error immediately without queuing a retry", func() { + retryW.WriteLogs([]*logspb.ResourceLogs{{}}) + + retryLSC.mu.Lock() + count := retryLSC.exportCount + retryLSC.mu.Unlock() + + Expect(count).To(Equal(1)) + Expect(buf).To(gbytes.Say("Write error:.*rate limited")) + }) + }) + + Context("when a trace export fails with Aborted", func() { + BeforeEach(func() { + retryTSC.responseErr = status.Error(codes.Aborted, "transaction aborted") + }) + + It("logs the error immediately without queuing a retry", func() { + retryW.WriteTrace([]*tracepb.ResourceSpans{{}}) + + retryTSC.mu.Lock() + count := retryTSC.exportCount + retryTSC.mu.Unlock() + + Expect(count).To(Equal(1)) + Expect(buf).To(gbytes.Say("Write error:.*transaction aborted")) + }) + }) + + Context("when the retry queue is full", func() { + BeforeEach(func() { + retryW.metricsRetry = make(chan retryItem, 1) + retryMSC.responseErr = status.Error(codes.Unavailable, "collector down") + }) + + It("logs a queue-full error for the overflow batch", func() { + retryW.WriteMetrics([]*metricspb.Metric{{Name: "m1"}}) + retryW.WriteMetrics([]*metricspb.Metric{{Name: "m2"}}) + Eventually(buf).Should(gbytes.Say("retry queue full")) + }) + }) + + Context("when the context is cancelled while retries are pending", func() { + BeforeEach(func() { + retryMSC.responseErr = status.Error(codes.Unavailable, "collector down") + }) + + It("stops the retry worker without logging a write error", func() { + retryW.WriteMetrics([]*metricspb.Metric{{Name: "test-metric"}}) + Eventually(func() int { + retryMSC.mu.Lock() + defer retryMSC.mu.Unlock() + return retryMSC.exportCount + }).Should(BeNumerically(">=", 1)) + + retryCancel() + // Allow goroutine to observe cancellation, then confirm no error was logged + // for the cancelled in-flight retry. + Consistently(buf, "50ms").ShouldNot(gbytes.Say("Write error")) + }) + }) + }) + Describe("Close", func() { It("cancels the gRPC context", func() { envelope := &loggregator_v2.Envelope{ @@ -1078,42 +1244,93 @@ var _ = Describe("Client", func() { }) type spyMetricsServiceClient struct { - requests chan *colmetricspb.ExportMetricsServiceRequest - response *colmetricspb.ExportMetricsServiceResponse - responseErr error - ctx context.Context + mu sync.Mutex + requests chan *colmetricspb.ExportMetricsServiceRequest + response *colmetricspb.ExportMetricsServiceResponse + responseErr error + responseErrs []error // dequeued on successive calls; falls back to responseErr when empty + exportCount int + ctx context.Context } func (c *spyMetricsServiceClient) Export(ctx context.Context, in *colmetricspb.ExportMetricsServiceRequest, opts ...grpc.CallOption) (*colmetricspb.ExportMetricsServiceResponse, error) { - c.requests <- in + c.mu.Lock() + c.exportCount++ + var err error + if len(c.responseErrs) > 0 { + err = c.responseErrs[0] + c.responseErrs = c.responseErrs[1:] + } else { + err = c.responseErr + } + c.mu.Unlock() + + select { + case c.requests <- in: + default: + } c.ctx = ctx - return c.response, c.responseErr + return c.response, err } type spyLogsServiceClient struct { - requests chan *collogspb.ExportLogsServiceRequest - response *collogspb.ExportLogsServiceResponse - responseErr error - ctx context.Context + mu sync.Mutex + requests chan *collogspb.ExportLogsServiceRequest + response *collogspb.ExportLogsServiceResponse + responseErr error + responseErrs []error + exportCount int + ctx context.Context } func (c *spyLogsServiceClient) Export(ctx context.Context, in *collogspb.ExportLogsServiceRequest, opts ...grpc.CallOption) (*collogspb.ExportLogsServiceResponse, error) { - c.requests <- in + c.mu.Lock() + c.exportCount++ + var err error + if len(c.responseErrs) > 0 { + err = c.responseErrs[0] + c.responseErrs = c.responseErrs[1:] + } else { + err = c.responseErr + } + c.mu.Unlock() + + select { + case c.requests <- in: + default: + } c.ctx = ctx - return c.response, c.responseErr + return c.response, err } type spyTraceServiceClient struct { - requests chan *coltracepb.ExportTraceServiceRequest - response *coltracepb.ExportTraceServiceResponse - responseErr error - ctx context.Context + mu sync.Mutex + requests chan *coltracepb.ExportTraceServiceRequest + response *coltracepb.ExportTraceServiceResponse + responseErr error + responseErrs []error + exportCount int + ctx context.Context } func (c *spyTraceServiceClient) Export(ctx context.Context, in *coltracepb.ExportTraceServiceRequest, opts ...grpc.CallOption) (*coltracepb.ExportTraceServiceResponse, error) { - c.requests <- in + c.mu.Lock() + c.exportCount++ + var err error + if len(c.responseErrs) > 0 { + err = c.responseErrs[0] + c.responseErrs = c.responseErrs[1:] + } else { + err = c.responseErr + } + c.mu.Unlock() + + select { + case c.requests <- in: + default: + } c.ctx = ctx - return c.response, c.responseErr + return c.response, err } func span(tsr *coltracepb.ExportTraceServiceRequest) *tracepb.Span { From 652ca444e850638800a583545dc1a8279b71b39c Mon Sep 17 00:00:00 2001 From: Wei Li Date: Thu, 11 Jun 2026 11:25:09 -0700 Subject: [PATCH 2/4] Move parameter values as constants --- src/pkg/otelcolclient/otelcolclient.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/pkg/otelcolclient/otelcolclient.go b/src/pkg/otelcolclient/otelcolclient.go index b1afbe7a1..f0d3a1ea6 100644 --- a/src/pkg/otelcolclient/otelcolclient.go +++ b/src/pkg/otelcolclient/otelcolclient.go @@ -25,6 +25,13 @@ import ( "google.golang.org/grpc/status" ) +const ( + defaultMaxRetries = 7 + defaultInitialRetryDelay = 1 * time.Second + defaultMaxRetryDelay = 30 * time.Second + defaultRetryQueueSize = 1024 +) + // retryItem holds a failed export closure and a count of retry attempts already made. type retryItem struct { exportFn func() error @@ -87,12 +94,12 @@ func NewGRPCWriter(addr string, tlsConfig *tls.Config, l *log.Logger) (*GRPCWrit ctx: ctx, cancel: cancel, l: l, - maxRetries: 7, - initialRetryDelay: 1 * time.Second, - maxRetryDelay: 30 * time.Second, - metricsRetry: make(chan retryItem, 1024), - logsRetry: make(chan retryItem, 1024), - tracesRetry: make(chan retryItem, 1024), + maxRetries: defaultMaxRetries, + initialRetryDelay: defaultInitialRetryDelay, + maxRetryDelay: defaultMaxRetryDelay, + metricsRetry: make(chan retryItem, defaultRetryQueueSize), + logsRetry: make(chan retryItem, defaultRetryQueueSize), + tracesRetry: make(chan retryItem, defaultRetryQueueSize), } go w.runRetryWorker(w.metricsRetry) go w.runRetryWorker(w.logsRetry) From 4c757fd25f0195caa7fa3005c8affb906c780ba0 Mon Sep 17 00:00:00 2001 From: Wei Li Date: Fri, 12 Jun 2026 14:39:02 -0700 Subject: [PATCH 3/4] Make OTel retry parameters configurable via env vars Introduce OTEL_RETRY_MAX_RETRIES and OTEL_RETRY_QUEUE_SIZE env vars so operators can tune the async retry behaviour without recompiling. The previous hard-coded values (7 retries, queue size 1024) are kept as defaults. Backoff delays (1 s initial, 30 s max) remain fixed constants as they do not need to be operator-facing. Updated retry log output to be less chatty. --- jobs/loggr-forwarder-agent-windows/monit | 2 + jobs/loggr-forwarder-agent-windows/spec | 8 +++ jobs/loggr-forwarder-agent/spec | 8 +++ .../templates/bpm.yml.erb | 3 + src/cmd/forwarder-agent/app/config.go | 11 ++++ .../forwarder-agent/app/forwarder_agent.go | 16 ++++-- src/pkg/otelcolclient/otelcolclient.go | 57 ++++++++++++------- src/pkg/otelcolclient/otelcolclient_test.go | 14 ++--- 8 files changed, 86 insertions(+), 33 deletions(-) diff --git a/jobs/loggr-forwarder-agent-windows/monit b/jobs/loggr-forwarder-agent-windows/monit index 93e2a72f1..60c362071 100644 --- a/jobs/loggr-forwarder-agent-windows/monit +++ b/jobs/loggr-forwarder-agent-windows/monit @@ -28,6 +28,8 @@ "EMIT_OTEL_TRACES" => "#{p("emit_otel_traces")}", "EMIT_OTEL_METRICS" => "#{p("emit_otel_metrics")}", "EMIT_OTEL_LOGS" => "#{p("emit_otel_logs")}", + "OTEL_RETRY_MAX_RETRIES" => "#{p("otel_retry.max_retries")}", + "OTEL_RETRY_QUEUE_SIZE" => "#{p("otel_retry.queue_size")}", "METRICS_PORT" => "#{p("metrics.port")}", "METRICS_CA_FILE_PATH" => "#{certs_dir}/metrics_ca.crt", "METRICS_CERT_FILE_PATH" => "#{certs_dir}/metrics.crt", diff --git a/jobs/loggr-forwarder-agent-windows/spec b/jobs/loggr-forwarder-agent-windows/spec index e85d3adc1..245cf66eb 100644 --- a/jobs/loggr-forwarder-agent-windows/spec +++ b/jobs/loggr-forwarder-agent-windows/spec @@ -49,6 +49,14 @@ properties: description: "Emit logs to downstream OpenTelemetry consumers" default: true + otel_retry.max_retries: + description: "Maximum number of retry attempts before a failed OTel Collector export batch is dropped" + default: 7 + + otel_retry.queue_size: + description: "Per-signal retry queue capacity (number of export batches that can be buffered per signal type)" + default: 1024 + tls.ca_cert: description: | TLS loggregator root CA certificate. It is required for key/cert diff --git a/jobs/loggr-forwarder-agent/spec b/jobs/loggr-forwarder-agent/spec index fadee9e24..af2c3737d 100644 --- a/jobs/loggr-forwarder-agent/spec +++ b/jobs/loggr-forwarder-agent/spec @@ -49,6 +49,14 @@ properties: description: "Emit logs to downstream OpenTelemetry consumers" default: true + otel_retry.max_retries: + description: "Maximum number of retry attempts before a failed OTel Collector export batch is dropped" + default: 7 + + otel_retry.queue_size: + description: "Per-signal retry queue capacity (number of export batches that can be buffered per signal type)" + default: 1024 + tls.ca_cert: description: | TLS loggregator root CA certificate. It is required for key/cert diff --git a/jobs/loggr-forwarder-agent/templates/bpm.yml.erb b/jobs/loggr-forwarder-agent/templates/bpm.yml.erb index 7e8f502f9..9ce12267c 100644 --- a/jobs/loggr-forwarder-agent/templates/bpm.yml.erb +++ b/jobs/loggr-forwarder-agent/templates/bpm.yml.erb @@ -34,6 +34,9 @@ "EMIT_OTEL_METRICS" => p("emit_otel_metrics"), "EMIT_OTEL_LOGS" => p("emit_otel_logs"), + "OTEL_RETRY_MAX_RETRIES" => p("otel_retry.max_retries"), + "OTEL_RETRY_QUEUE_SIZE" => p("otel_retry.queue_size"), + "METRICS_PORT" => "#{p("metrics.port")}", "METRICS_CA_FILE_PATH" => "#{certs_dir}/metrics_ca.crt", "METRICS_CERT_FILE_PATH" => "#{certs_dir}/metrics.crt", diff --git a/src/cmd/forwarder-agent/app/config.go b/src/cmd/forwarder-agent/app/config.go index 022e27bd4..1b96c767a 100644 --- a/src/cmd/forwarder-agent/app/config.go +++ b/src/cmd/forwarder-agent/app/config.go @@ -19,6 +19,12 @@ type GRPC struct { CipherSuites []string `env:"AGENT_CIPHER_SUITES, report"` } +// OtelRetry holds tunable parameters for the OTel Collector gRPC retry logic. +type OtelRetry struct { + MaxRetries int `env:"OTEL_RETRY_MAX_RETRIES, report"` + RetryQueueSize int `env:"OTEL_RETRY_QUEUE_SIZE, report"` +} + // Config holds the configuration for the forwarder agent type Config struct { UseRFC3339 bool `env:"USE_RFC3339"` @@ -33,6 +39,7 @@ type Config struct { EmitOTelTraces bool `env:"EMIT_OTEL_TRACES, report"` EmitOTelMetrics bool `env:"EMIT_OTEL_METRICS, report"` EmitOTelLogs bool `env:"EMIT_OTEL_LOGS, report"` + OtelRetry OtelRetry } // LoadConfig will load the configuration for the forwarder agent from the @@ -44,6 +51,10 @@ func LoadConfig() Config { Host: "127.0.0.1", Port: 3458, }, + OtelRetry: OtelRetry{ + MaxRetries: 7, + RetryQueueSize: 1024, + }, } if err := envstruct.Load(&cfg); err != nil { panic(fmt.Sprintf("Failed to load config from environment: %s", err)) diff --git a/src/cmd/forwarder-agent/app/forwarder_agent.go b/src/cmd/forwarder-agent/app/forwarder_agent.go index 0cf889b80..db3315cb1 100644 --- a/src/cmd/forwarder-agent/app/forwarder_agent.go +++ b/src/cmd/forwarder-agent/app/forwarder_agent.go @@ -44,6 +44,7 @@ type ForwarderAgent struct { emitOTelTraces bool emitOTelMetrics bool emitOTelLogs bool + otelRetry OtelRetry } type Metrics interface { @@ -77,6 +78,7 @@ func NewForwarderAgent( emitOTelTraces: cfg.EmitOTelTraces, emitOTelMetrics: cfg.EmitOTelMetrics, emitOTelLogs: cfg.EmitOTelLogs, + otelRetry: cfg.OtelRetry, } } @@ -100,7 +102,7 @@ func (s *ForwarderAgent) Run() { })) dests := downstreamDestinations(s.downstreamFilePattern, s.log) - writers := downstreamWriters(dests, s.grpc, s.m, s.emitOTelTraces, s.emitOTelMetrics, s.emitOTelLogs, s.log) + writers := downstreamWriters(dests, s.grpc, s.m, s.emitOTelTraces, s.emitOTelMetrics, s.emitOTelLogs, s.otelRetry, s.log) tagger := egress_v2.NewTagger(s.tags) ew := egress_v2.NewEnvelopeWriter( multiWriter{writers: writers}, @@ -213,13 +215,13 @@ func downstreamDestinations(pattern string, l *log.Logger) []destination { return dests } -func downstreamWriters(dests []destination, grpc GRPC, m Metrics, emitOTelTraces, emitOTelMetrics, emitOTelLogs bool, l *log.Logger) []Writer { +func downstreamWriters(dests []destination, grpc GRPC, m Metrics, emitOTelTraces, emitOTelMetrics, emitOTelLogs bool, otelRetry OtelRetry, l *log.Logger) []Writer { var writers []Writer for _, d := range dests { var w Writer switch d.Protocol { case "otelcol": - w = otelCollectorClient(d, grpc, m, emitOTelTraces, emitOTelMetrics, emitOTelLogs, l) + w = otelCollectorClient(d, grpc, m, emitOTelTraces, emitOTelMetrics, emitOTelLogs, otelRetry, l) default: w = loggregatorClient(d, grpc, m, l) } @@ -228,7 +230,7 @@ func downstreamWriters(dests []destination, grpc GRPC, m Metrics, emitOTelTraces return writers } -func otelCollectorClient(dest destination, grpc GRPC, m Metrics, emitTraces, emitMetrics, emitLogs bool, l *log.Logger) Writer { +func otelCollectorClient(dest destination, grpc GRPC, m Metrics, emitTraces, emitMetrics, emitLogs bool, otelRetry OtelRetry, l *log.Logger) Writer { clientCreds, err := tlsconfig.Build( tlsconfig.WithInternalServiceDefaults(), tlsconfig.WithIdentityFromFile(grpc.CertFile, grpc.KeyFile), @@ -242,7 +244,11 @@ func otelCollectorClient(dest destination, grpc GRPC, m Metrics, emitTraces, emi occl := log.New(l.Writer(), fmt.Sprintf("[OTEL COLLECTOR CLIENT] -> %s: ", dest.Ingress), l.Flags()) - w, err := otelcolclient.NewGRPCWriter(dest.Ingress, clientCreds, occl) + writerCfg := otelcolclient.GRPCWriterConfig{ + MaxRetries: otelRetry.MaxRetries, + RetryQueueSize: otelRetry.RetryQueueSize, + } + w, err := otelcolclient.NewGRPCWriter(dest.Ingress, clientCreds, writerCfg, occl) if err != nil { l.Fatalf("Failed to create OTel Collector gRPC writer for %s: %s", dest.Ingress, err) } diff --git a/src/pkg/otelcolclient/otelcolclient.go b/src/pkg/otelcolclient/otelcolclient.go index f0d3a1ea6..51be9e9c1 100644 --- a/src/pkg/otelcolclient/otelcolclient.go +++ b/src/pkg/otelcolclient/otelcolclient.go @@ -26,12 +26,18 @@ import ( ) const ( - defaultMaxRetries = 7 - defaultInitialRetryDelay = 1 * time.Second - defaultMaxRetryDelay = 30 * time.Second - defaultRetryQueueSize = 1024 + retryInitialDelay = 1 * time.Second + retryMaxDelay = 30 * time.Second ) +// GRPCWriterConfig holds tunable parameters for the GRPCWriter retry behaviour. +type GRPCWriterConfig struct { + // MaxRetries is the maximum number of retry attempts before a batch is dropped. + MaxRetries int + // RetryQueueSize is the per-signal channel capacity for pending retry batches. + RetryQueueSize int +} + // retryItem holds a failed export closure and a count of retry attempts already made. type retryItem struct { exportFn func() error @@ -63,7 +69,10 @@ type GRPCWriter struct { maxRetryDelay time.Duration // Per-signal queues for async retry workers. Items are enqueued by withRetry - // and consumed by runRetryWorker goroutines started in NewGRPCWriter. + // and consumed by runRetryWorker goroutines started in NewGRPCWriter. The withRetry + // method enqueues a failed batch to a per-signal buffered channel and returns + // immediately, releasing the SignalBatcher mutex so the DiodeWriter ring + // buffer continues to drain without stalling. // Sized to absorb the full retry window: 7 attempts over 91 s (1+2+4+8+16+30+30) // at the 100 ms flush interval produces at most 910 batches. 1024 clears that // with a small margin. A nil channel disables async retry (used in tests that @@ -74,7 +83,7 @@ type GRPCWriter struct { } // NewGRPCWriter dials the provided gRPC address and returns a *GRPCWriter. -func NewGRPCWriter(addr string, tlsConfig *tls.Config, l *log.Logger) (*GRPCWriter, error) { +func NewGRPCWriter(addr string, tlsConfig *tls.Config, cfg GRPCWriterConfig, l *log.Logger) (*GRPCWriter, error) { cc, err := grpc.NewClient(addr, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) if err != nil { return nil, err @@ -94,16 +103,16 @@ func NewGRPCWriter(addr string, tlsConfig *tls.Config, l *log.Logger) (*GRPCWrit ctx: ctx, cancel: cancel, l: l, - maxRetries: defaultMaxRetries, - initialRetryDelay: defaultInitialRetryDelay, - maxRetryDelay: defaultMaxRetryDelay, - metricsRetry: make(chan retryItem, defaultRetryQueueSize), - logsRetry: make(chan retryItem, defaultRetryQueueSize), - tracesRetry: make(chan retryItem, defaultRetryQueueSize), - } - go w.runRetryWorker(w.metricsRetry) - go w.runRetryWorker(w.logsRetry) - go w.runRetryWorker(w.tracesRetry) + maxRetries: cfg.MaxRetries, + initialRetryDelay: retryInitialDelay, + maxRetryDelay: retryMaxDelay, + metricsRetry: make(chan retryItem, cfg.RetryQueueSize), + logsRetry: make(chan retryItem, cfg.RetryQueueSize), + tracesRetry: make(chan retryItem, cfg.RetryQueueSize), + } + go w.runRetryWorker("metrics", w.metricsRetry) + go w.runRetryWorker("logs", w.logsRetry) + go w.runRetryWorker("traces", w.tracesRetry) cancel = nil return w, nil } @@ -217,7 +226,7 @@ func drainRetryQueue(dst []retryItem, src <-chan retryItem) []retryItem { // The shared pool delay resets to initialRetryDelay each time the pool drains // to empty. Items that exhaust maxRetries are logged and discarded. The // goroutine exits silently when the writer's context is cancelled. -func (w *GRPCWriter) runRetryWorker(queue <-chan retryItem) { +func (w *GRPCWriter) runRetryWorker(signal string, queue <-chan retryItem) { var pool []retryItem delay := w.initialRetryDelay @@ -227,7 +236,6 @@ func (w *GRPCWriter) runRetryWorker(queue <-chan retryItem) { pool = drainRetryQueue(pool, queue) if prevLen == 0 && len(pool) > 0 { delay = w.initialRetryDelay - w.l.Println("New item added to empty pool, delay set to", delay) } if len(pool) == 0 { @@ -238,7 +246,6 @@ func (w *GRPCWriter) runRetryWorker(queue <-chan retryItem) { case item := <-queue: pool = append(pool, item) delay = w.initialRetryDelay - w.l.Println("New item added to empty pool, delay set to", delay) } pool = drainRetryQueue(pool, queue) } @@ -255,6 +262,7 @@ func (w *GRPCWriter) runRetryWorker(queue <-chan retryItem) { // Attempt every item in the pool; keep the ones that still need more retries. var remaining []retryItem + var lastError error for _, item := range pool { if isContextError(w.ctx.Err()) { return @@ -268,16 +276,23 @@ func (w *GRPCWriter) runRetryWorker(queue <-chan retryItem) { } item.attempts++ if !isRetryable(err) || item.attempts >= w.maxRetries { - w.l.Println("Write error:", err) + w.l.Printf("Dropping %s batch after %d attempts: %v", signal, item.attempts, err) continue } + lastError = err remaining = append(remaining, item) } + + if len(remaining) > 0 { + w.l.Printf("Retrying %d %s batches in %s, last err: %v", len(remaining), signal, delay, lastError) + } + if len(pool) > 0 && len(remaining) == 0 { + w.l.Printf("%s retry pool drained after recovery", signal) + } pool = remaining if len(pool) == 0 { delay = w.initialRetryDelay - w.l.Println("Pool is empty, delay set to", delay) } else { delay = min(delay*2, w.maxRetryDelay) } diff --git a/src/pkg/otelcolclient/otelcolclient_test.go b/src/pkg/otelcolclient/otelcolclient_test.go index 0ae14a3e7..a220575bd 100644 --- a/src/pkg/otelcolclient/otelcolclient_test.go +++ b/src/pkg/otelcolclient/otelcolclient_test.go @@ -1097,9 +1097,9 @@ var _ = Describe("Client", func() { logsRetry: make(chan retryItem, 16), tracesRetry: make(chan retryItem, 16), } - go retryW.runRetryWorker(retryW.metricsRetry) - go retryW.runRetryWorker(retryW.logsRetry) - go retryW.runRetryWorker(retryW.tracesRetry) + go retryW.runRetryWorker("metrics", retryW.metricsRetry) + go retryW.runRetryWorker("logs", retryW.logsRetry) + go retryW.runRetryWorker("traces", retryW.tracesRetry) }) AfterEach(func() { @@ -1131,10 +1131,10 @@ var _ = Describe("Client", func() { retryMSC.responseErr = status.Error(codes.Unavailable, "collector down") }) - It("logs a write error after the final retry attempt", func() { - retryW.WriteMetrics([]*metricspb.Metric{{Name: "test-metric"}}) - Eventually(buf).Should(gbytes.Say("Write error:.*collector down")) - }) + It("logs a drop message after the final retry attempt", func() { + retryW.WriteMetrics([]*metricspb.Metric{{Name: "test-metric"}}) + Eventually(buf).Should(gbytes.Say("Dropping metrics batch after.*collector down")) + }) }) Context("when the error is not retryable", func() { From bd5571eeff1a49ea2d6b861ac1b5a53977374f9c Mon Sep 17 00:00:00 2001 From: Wei Li Date: Fri, 12 Jun 2026 14:39:02 -0700 Subject: [PATCH 4/4] Make OTel retry parameters configurable via env vars Introduce OTEL_RETRY_MAX_RETRIES and OTEL_RETRY_QUEUE_SIZE env vars so operators can tune the async retry behaviour without recompiling. The previous hard-coded values (7 retries, queue size 1024) are kept as defaults. Backoff delays (1 s initial, 30 s max) remain fixed constants as they do not need to be operator-facing. Updated retry log output to be less chatty. --- src/go.sum | 110 ----------------------------------------------------- 1 file changed, 110 deletions(-) diff --git a/src/go.sum b/src/go.sum index 9e39a02f1..757a383bf 100644 --- a/src/go.sum +++ b/src/go.sum @@ -1,11 +1,3 @@ -cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= -cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= -cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= -cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= -cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= -cloud.google.com/go/kms v1.31.0/go.mod h1:YIyXZym11R5uovJJt4oN5eUL3oPmirF3yKeIh6QAf4U= -cloud.google.com/go/longrunning v0.9.0/go.mod h1:pkTz846W7bF4o2SzdWJ40Hu0Re+UoNT6Q5t+igIcb8E= code.cloudfoundry.org/go-batching v0.0.0-20260526123032-013946b96a09 h1:0UJG1LYBpGC7rrVjFvmSkfuOYVqiI0DUmLJOktGiU3c= code.cloudfoundry.org/go-batching v0.0.0-20260526123032-013946b96a09/go.mod h1:PoY0RDyVol/cGj+v9DMBznRcG5FgcWD91Uy2iqN/Caw= code.cloudfoundry.org/go-diodes v0.0.0-20260526122959-0284fcb5ac88 h1:9HNd8DjiaHrHwiblgczOwfufgVBlDozkgVv5FxqZcts= @@ -18,62 +10,22 @@ code.cloudfoundry.org/go-metric-registry v0.0.0-20260602223408-eae7443fd7fa h1:v code.cloudfoundry.org/go-metric-registry v0.0.0-20260602223408-eae7443fd7fa/go.mod h1:AKZJZ2b22vZTfc20eev4vao3Z5HUeBCRy5pSAJrijtY= code.cloudfoundry.org/tlsconfig v0.58.0 h1:UGMvB95Q1hkBVEHEAg+8i6pFBmLPRo5XaD+xtJ4E6DQ= code.cloudfoundry.org/tlsconfig v0.58.0/go.mod h1:C7+gD2NiT/00LliHJOGJhpjO9myczswc3uvv2++payw= -dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1/go.mod h1:pzBXCYn05zvYIrwLgtK8Ap8QcjRg+0i76tMQdWN6wOk= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0/go.mod h1:i2h9fsTFKZorh8RdV2IcSUf/Qj98GlTkrTvUbX/s8as= -github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= -github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= -github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= -github.com/ThalesGroup/crypto11 v1.6.0/go.mod h1:H6LRjN5R5SHxTrLqGNteisLDI0/IC6+SGx1pHtbwizE= -github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= -github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apoydence/eachers v0.0.0-20181020210610-23942921fe77/go.mod h1:bXvGk6IkT1Agy7qzJ+DjIw/SJ1AaB3AvAuMDVV+Vkoo= -github.com/aws/aws-sdk-go-v2 v1.41.11/go.mod h1:iiUX27gOXRuYaoeUVXhUpPwjJHzISfPAjjcuhUbLSVs= -github.com/aws/aws-sdk-go-v2/config v1.32.22/go.mod h1:0+H+0nPKbvWltf5vSIGkApv+hGbaQ4FfwTjGIYQREcw= -github.com/aws/aws-sdk-go-v2/credentials v1.19.21/go.mod h1:UE8+9t5zudFwu5k5ShC1PKArVEdOkQQdCXIHQAVNUcU= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.27/go.mod h1:ISGSFNbOHRS+JV/17yStzRTPBUHHqF92kCpRLLyH3Nk= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.27/go.mod h1:QV9IVIopJ1dpQUno0f9VYDUwOEjj8u0iEJ4JiZVre3Y= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.27/go.mod h1:x0rldpsnUQaQIs4Rh+Vwm9Z/0vI6BxadGtsgJfZFb8s= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.28/go.mod h1:oTdbDr+BMs7gAYrNpD0LDTyqQfv6yOYgTDv46+xbwFY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.11/go.mod h1:XG68qW+YLLFH0vnSDCou43Cgj5TeAG83O5NRSJgt04Y= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.27/go.mod h1:p7hwgbwompjCRNTdB3ytlldddNt1rDBgVVMqWEVG1II= -github.com/aws/aws-sdk-go-v2/service/kms v1.53.2/go.mod h1:ZNloshpT3zBMbDUwsbbwYP46ADvoJU9/9WGX1WYegwg= -github.com/aws/aws-sdk-go-v2/service/signin v1.1.3/go.mod h1:WhO1EH3phjFWValQDsExaxncgEWJsHeoTvuyQAj3jwU= -github.com/aws/aws-sdk-go-v2/service/sso v1.31.1/go.mod h1:tEL1hqCrkgwrDVL04HuLxz1SLUXdh+4kKhWv1pXKeiY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.4/go.mod h1:0x10Wy0dVS4Gn552xhHY5th2QdYpfJf44EsfyYGV194= -github.com/aws/aws-sdk-go-v2/service/sts v1.43.1/go.mod h1:t01JURC8Fe5M+7R1K0vzIZ2NT04HqvZR+FjlHrHDT2A= -github.com/aws/smithy-go v1.27.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/cloudfoundry/dropsonde v1.1.0 h1:nerhj8K0heOsv/U/Ddou5Esw56YlNeHHJH6MP9QlACQ= github.com/cloudfoundry/dropsonde v1.1.0/go.mod h1:OrkxsBrAvM8X0Ve9vaSNKLR+/Jeohu3+J0M4JEaTmnM= -github.com/cloudfoundry/gosteno v0.0.0-20150423193413-0c8581caea35/go.mod h1:3YBPUR85RIrvaUTdA1dL38YSp6s3OHu1xrWLkGt2Mog= github.com/cloudfoundry/sonde-go v0.0.0-20260526083715-66f310f13c26 h1:DHWZw6RGiApykMdpy4d1t0OApzautzvnJI60H2B5OSM= github.com/cloudfoundry/sonde-go v0.0.0-20260526083715-66f310f13c26/go.mod h1:rR3cyMIEfcaknKHjUDMf7Cuq5gn9Aiqvzz/gqsnsQBI= -github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= -github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= -github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= -github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= @@ -82,47 +34,26 @@ github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01 github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM= github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= -github.com/go-jose/go-jose/v3 v3.0.5/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= -github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-piv/piv-go/v2 v2.6.0/go.mod h1:gcsS2WGbToZk5PwtsCIlCWzV/R+r/wN+Xi+6O6IlU3c= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= -github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/certificate-transparency-go v1.1.2/go.mod h1:3OL+HKDqHPUfdKrHVQxO6T8nDLO0HF7LRTlkIWXaWvQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= -github.com/google/go-tpm-tools v0.4.8/go.mod h1:4DfiOtiS1KppJjwf1+tqtW4K3PrCJjAAqFKj/TYTJKg= -github.com/google/go-tspi v0.3.0/go.mod h1:xfMGI3G0PhxCdNVcYr1C4C+EizojDg/TXuX5by8CiHI= github.com/google/pprof v0.0.0-20260604005048-7023385849c0 h1:h1QTMDl6q9wDvDCJVpKQSjgleGFYnd2fOxmg2K+6BGE= github.com/google/pprof v0.0.0-20260604005048-7023385849c0/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= -github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.16/go.mod h1:9Yb0eAkH/Xqhvv3zbeKf/+wMJqCeocWc6KIhDvEAuYE= -github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= -github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= -github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/ianlancetaylor/demangle v0.0.0-20250417193237-f615e6bd150b/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -131,38 +62,24 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2 h1:yVCLo4+ACVroOEr4iFU1iH46Ldlzz2rTuu18Ra7M8sU= github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2/go.mod h1:VzB2VoMh1Y32/QqDfg9ZJYHj99oM4LiGtqPZydTiQSQ= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= -github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= -github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= -github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.29.0 h1:rfh+ZFjgJhYWRoIqVf3Uwx/W20yLrcrE2h2GmYVRaag= github.com/onsi/ginkgo/v2 v2.29.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.41.0 h1:OwKp4pXNgVxf6sCplzYo794OFNuoL2q2SBMU5NSWOjA= github.com/onsi/gomega v1.41.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= -github.com/peterbourgon/diskv/v3 v3.0.1/go.mod h1:kJ5Ny7vLdARGU3WUuy6uzO6T0nb/2gWcT1JiBvRmb5o= -github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/poy/eachers v0.0.0-20181020210610-23942921fe77/go.mod h1:x1vqpbcMW9T/KRcQ4b48diSiSVtYgvwQ5xzDByEg4WE= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -171,24 +88,14 @@ github.com/prometheus/common v0.68.1 h1:omjRRl4QP4komogpXuhfeOiisQg7xdy8VM1UY+pS github.com/prometheus/common v0.68.1/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/schollz/jsonstore v1.1.0/go.mod h1:15c6+9guw8vDRyozGjN3FoILt0wpruJk9Pi66vjaZfg= github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= -github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/smallstep/go-attestation v0.4.4-0.20260603212853-e1a87a0b07d9/go.mod h1:vNAduivU014fubg6ewygkAvQC0IQVXqdc8vaGl/0er4= -github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/square/certstrap v1.3.0 h1:N9P0ZRA+DjT8pq5fGDj0z3FjafRKnBDypP0QHpMlaAk= github.com/square/certstrap v1.3.0/go.mod h1:wGZo9eE1B7WX2GKBn0htJ+B3OuRl2UsdCFySNooy9hU= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/thales-e-security/pool v0.0.2/go.mod h1:qtpMm2+thHtqhLzTwgDBj/OuNnMpupY8mv0Phz0gjhU= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -197,20 +104,14 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/urfave/cli v1.22.9/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.71.0 h1:tepR7H+Guh9VUqxxcPggYi8R3lGUu2Rsdh+z7/FCY3k= github.com/valyala/fasthttp v1.71.0/go.mod h1:z1sDUvOShhXq/C9mwH/fSm1Vb71tUJwmQdgkBrBNwnA= -github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= @@ -227,7 +128,6 @@ go.step.sm/crypto v0.82.0 h1:JOT8b/7Jh4My3mxE4U7UkuaN2sUGkZ8fnjznXaTGoRE= go.step.sm/crypto v0.82.0/go.mod h1:qyLTv666WJ6ImFPUjljux+684Y/GGYUjAZcKCnc6yBs= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= @@ -238,23 +138,18 @@ golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= -golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6/go.mod h1:Eqhaxk/wZsWEH8CRxLwj6xzEJbz7k1EFGqx7nyCoabE= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= -golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/api v0.283.0/go.mod h1:6Wssta4c5n9qHq5CBhmlai5h/PUa1djdDAIhYEHyvcM= -google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= @@ -270,8 +165,3 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= -modernc.org/libc v1.72.3/go.mod h1:dn0dZNnnn1clLyvRxLxYExxiKRZIRENOfqQ8XEeg4Qs= -modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= -modernc.org/sqlite v1.51.0/go.mod h1:tcNzv5p84E0skkmJn038y+hWJbLQXQqEnQfeh5r2JLM=