From 725981718576f0184e86899ca69d6ccc141df356 Mon Sep 17 00:00:00 2001 From: kiro-agent Date: Thu, 30 Jul 2026 08:03:32 +0000 Subject: [PATCH] docs: add recovery request queue configuration guide Signed-off-by: kiro-agent --- astro.config.mjs | 1 + .../how-to/connections/recovery-queue.mdx | 87 +++++++++++++++++++ .../connections/resilience-best-practices.mdx | 5 ++ 3 files changed, 93 insertions(+) create mode 100644 src/content/docs/how-to/connections/recovery-queue.mdx diff --git a/astro.config.mjs b/astro.config.mjs index 9bda7bc7..0f7b454a 100755 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -180,6 +180,7 @@ export default defineConfig({ "how-to/connections/configure-lazy-connection", "how-to/connections/limit-inflight-requests", "how-to/connections/read-strategy", + "how-to/connections/recovery-queue", "how-to/connections/resilience-best-practices", "how-to/connections/timeouts-and-reconnect-strategy", ], diff --git a/src/content/docs/how-to/connections/recovery-queue.mdx b/src/content/docs/how-to/connections/recovery-queue.mdx new file mode 100644 index 00000000..26dcce08 --- /dev/null +++ b/src/content/docs/how-to/connections/recovery-queue.mdx @@ -0,0 +1,87 @@ +--- +title: Recovery Request Queue +description: Buffer requests during cluster topology-refresh reconnects instead of failing them immediately +--- + +import { Tabs, TabItem } from '@astrojs/starlight/components'; + +When a cluster client detects a circular MOVED redirect, it enters a recovery state to refresh the cluster topology. During this window, incoming requests would normally fail immediately with a `ClientError("Connection in recovery")`. The **recovery request queue** introduces a bounded buffer that holds those requests and retries them transparently once reconnection completes. + +## Configuration + +Configure the recovery request queue size through the cluster client configuration. + + + + :::note + This feature is not yet available in GLIDE C#. + ::: + + + + ```go + // Buffer up to 2000 requests during cluster recovery + clientConfig := config.NewClusterClientConfiguration(). + WithAddress(&config.NodeAddress{Host: "localhost", Port: 6379}). + WithRecoveryRequestsQueueSize(2000) + ``` + + + + ```java + // Buffer up to 2000 requests during cluster recovery + GlideClusterClientConfiguration config = GlideClusterClientConfiguration.builder() + .address(NodeAddress.builder().host("localhost").port(6379).build()) + .recoveryRequestsQueueSize(2000) + .build(); + ``` + + + + ```typescript + // Buffer up to 2000 requests during cluster recovery + const config: GlideClusterClientConfiguration = { + addresses: [{ host: "localhost", port: 6379 }], + recoveryRequestsQueueSize: 2000, + }; + ``` + + + + :::note + This feature is not yet available in GLIDE PHP. + ::: + + + + ```python + # Buffer up to 2000 requests during cluster recovery + config = GlideClusterClientConfiguration( + addresses=[NodeAddress(host="localhost", port=6379)], + recovery_requests_queue_size=2000, + ) + ``` + + + +## How It Works + +1. **Trigger:** The cluster client detects that the topology has changed when it encounters a circular MOVED redirect (a redirect loop indicating the cached slot map is stale). The client enters a recovery state to refresh its view of the cluster. +2. **Buffering:** During recovery, incoming requests are placed into a bounded FIFO queue instead of being failed immediately. This happens transparently — callers simply experience a brief delay rather than an error. +3. **Dispatch:** When recovery completes and the topology is refreshed, buffered requests are dispatched in FIFO order. Each request retains its full retry budget, so transient routing issues after recovery are handled normally. +4. **Overflow:** If the queue is full (i.e., the number of buffered requests reaches the configured cap), additional requests are failed immediately with an error. This prevents unbounded memory growth during prolonged recovery windows. + +## When to Adjust + +| Scenario | Recommendation | +|----------|---------------| +| High-throughput workloads with long recovery windows | Increase the queue size to avoid dropping requests during extended topology refreshes | +| Memory-constrained environments | Decrease the queue size to limit memory usage during recovery | +| Application handles retries at a higher level | Set to `0` to disable the queue and restore fail-fast behavior | +| Short-lived bursts with fast recovery | The default of 1000 is typically sufficient | + +**Disabling the queue:** Setting `recovery_requests_queue_size` to `0` restores the previous behavior where requests arriving during recovery are failed immediately. This is useful when your application already implements retry logic above the client layer and you prefer immediate feedback over transparent buffering. + +## Defaults + +The default recovery request queue size is **1000**. This applies only to cluster clients (`GlideClusterClient`). Standalone clients use a different reconnection mechanism and do not encounter circular MOVED redirects, so this option has no effect on them. diff --git a/src/content/docs/how-to/connections/resilience-best-practices.mdx b/src/content/docs/how-to/connections/resilience-best-practices.mdx index b744d702..55d04971 100644 --- a/src/content/docs/how-to/connections/resilience-best-practices.mdx +++ b/src/content/docs/how-to/connections/resilience-best-practices.mdx @@ -26,6 +26,7 @@ config = GlideClusterClientConfiguration( addresses=[NodeAddress(host="localhost", port=6379)], request_timeout=500, inflight_requests_limit=1000, + recovery_requests_queue_size=1000, client_circuit_breaker=ClientCircuitBreakerConfiguration(), ) client = await GlideClusterClient.create(config) @@ -45,6 +46,7 @@ GlideClusterClient client = GlideClusterClient.createClient( .address(NodeAddress.builder().host("localhost").port(6379).build()) .requestTimeout(500) .inflightRequestsLimit(1000) + .recoveryRequestsQueueSize(1000) .clientCircuitBreakerConfiguration( ClientCircuitBreakerConfiguration.builder().build()) .build() @@ -61,6 +63,7 @@ const client = await GlideClusterClient.createClient({ addresses: [{ host: "localhost", port: 6379 }], requestTimeout: 500, inflightRequestsLimit: 1000, + recoveryRequestsQueueSize: 1000, clientCircuitBreaker: {}, }); ``` @@ -80,6 +83,7 @@ clientConfig := config.NewClusterClientConfiguration(). WithAddress(&config.NodeAddress{Host: "localhost", Port: 6379}). WithRequestTimeout(500 * time.Millisecond). WithInflightRequestsLimit(1000). + WithRecoveryRequestsQueueSize(1000). WithClientCircuitBreaker(&config.ClientCircuitBreakerConfiguration{}) client, err := glide.NewClusterClient(clientConfig) @@ -189,6 +193,7 @@ Cluster clients additionally benefit from: * Per-node routing (commands to healthy nodes are unaffected by one bad node) * Topology refresh and failover detection (periodic, enabled by default) * Automatic MOVED/ASK redirect handling +* Recovery request queue: buffers requests during topology-refresh reconnects instead of failing them immediately (see [Recovery Request Queue](/how-to/connections/recovery-queue)) * `refreshTopologyFromInitialNodes`: when enabled, falls back to seed nodes if existing cluster nodes are unreachable during topology refresh (important when the failed node is the one the client uses for discovery). This is set via the advanced cluster client configuration. For standalone deployments, the client-wide circuit breaker is the primary protection against a single unresponsive server causing thread pile-up.