From 0781ad7230db6b6f7742550419e92149e2a3adba Mon Sep 17 00:00:00 2001 From: eric-liu-nvidia Date: Wed, 1 Jul 2026 01:04:58 +0800 Subject: [PATCH 01/11] feat(latency-service): caller_required credential policy + per-model upstream-attempts metric --- .agents/skills/switchyard-lib-core/SKILL.md | 2 +- examples/latency_service_caller_required.yaml | 70 +++++++ .../backends/latency_service_llm_backend.py | 113 ++++++++++- .../config/latency_service_backend_config.py | 21 ++- switchyard/lib/proxy_context.py | 5 +- switchyard/lib/request_metadata.py | 52 ++++- tests/test_latency_service_llm_backend.py | 177 ++++++++++++++++++ tests/test_request_metadata.py | 84 ++++++++- 8 files changed, 505 insertions(+), 19 deletions(-) create mode 100644 examples/latency_service_caller_required.yaml diff --git a/.agents/skills/switchyard-lib-core/SKILL.md b/.agents/skills/switchyard-lib-core/SKILL.md index 7a9933a2..15d472ff 100644 --- a/.agents/skills/switchyard-lib-core/SKILL.md +++ b/.agents/skills/switchyard-lib-core/SKILL.md @@ -43,7 +43,7 @@ right validation set. If the change is driven by a launcher need, also read | A fixed-path endpoint contributed by per-route components | Set `Endpoint.register_once = True`; `build_switchyard_app(...)` mounts the first instance while still running every component's lifecycle. Leave the default `False` for configurable endpoint classes that may mount distinct instances. | | Per-endpoint attribution on `/metrics` for a Python backend that can't be wrapped by `StatsLlmBackend` | Set `ctx.selected_model = endpoint_id` before returning the response. Also set `ctx.backend_call_latency_ms = upstream_call_ms` so the response processor can compute routing overhead. `LatencyServiceLLMBackend.call` is the reference. | | State metrics on `/metrics` | Register a `PrometheusEmitter` via `switchyard.lib.endpoints.prometheus_emitter.register(...)` and unregister on `shutdown()`. This is for backend-owned state, not request-flow counters. | -| Error-rate / retry-recovery counters | Use `switchyard.lib.endpoints.outcome_metrics`. FastAPI middleware records client outcomes. A retrying Python backend records each upstream attempt itself (and `record_retry_recovered()`) and sets `CTX_UPSTREAM_ATTEMPTS_RECORDED` so the endpoint skips its fallback — `LatencyServiceLLMBackend.call` is the reference. Single-attempt backends (Rust native / passthrough / multi) record nothing themselves; the endpoint fallback (`record_upstream_attempt_success` / `record_upstream_attempt_failure` in `upstream_error.py`, called from `dispatch_chat_request` and `handle_chain_exception`) counts their one attempt. Don't add a `model` label — these counters are layer-aggregate. Keep labels bounded. | +| Error-rate / retry-recovery counters | Use `switchyard.lib.endpoints.outcome_metrics`. FastAPI middleware records client outcomes. A retrying Python backend records each upstream attempt itself (and `record_retry_recovered()`) and sets `CTX_UPSTREAM_ATTEMPTS_RECORDED` so the endpoint skips its fallback — `LatencyServiceLLMBackend.call` is the reference. Single-attempt backends (Rust native / passthrough / multi) record nothing themselves; the endpoint fallback (`record_upstream_attempt_success` / `record_upstream_attempt_failure` in `upstream_error.py`, called from `dispatch_chat_request` and `handle_chain_exception`) counts their one attempt. Don't add a `model` label — these counters are layer-aggregate. Keep labels bounded. For a per-model error breakdown, emit a backend-owned counter via the `PrometheusEmitter` instead (bounded by the configured model set) — `LatencyServiceLLMBackend._render_prometheus_lines` exposes `switchyard_latency_upstream_attempts_total{requested_model,upstream_model,outcome,code}` next to the aggregate, leaving the shared counter model-free. | | Per-event error log | Use `switchyard.lib.endpoints.upstream_error_log.log_upstream_attempt_failure(...)` on the failure path. Events belong in logs/traces, not Prometheus sample timestamps. | | CLI launcher integration | Build one profile-backed `SwitchyardApp` with `build_tier_passthrough_switchyard(...)` for single-target mode, or merge route YAML with `load_route_bundle_table(...)`. Hand the result to `build_switchyard_app`. | | A new preset | Put preset helpers beside the profile config they produce, under `switchyard/lib/profiles/`. Presets should return typed config objects, not runnable chains. | diff --git a/examples/latency_service_caller_required.yaml b/examples/latency_service_caller_required.yaml new file mode 100644 index 00000000..6a829514 --- /dev/null +++ b/examples/latency_service_caller_required.yaml @@ -0,0 +1,70 @@ +# ============================================================================= +# Latency-aware routing with per-caller key attribution (BYO key, required) +# ============================================================================= +# +# Use this when every upstream call must be billed to the *caller's* own API key +# (e.g. per-user Inference Hub spend attribution) and the service-level key must +# NEVER authenticate caller inference. +# +# What `credential_policy: caller_required` does: +# * The caller key is read from the `x-switchyard-api-key` request header +# (preferred), then `Authorization: Bearer` / `x-api-key`. +# * Present -> forwarded upstream as `Authorization: Bearer `; spend is +# attributed to that caller. +# * Missing -> the request is rejected with HTTP 401 BEFORE any upstream call; +# the configured endpoint key is never used. +# (Applies to every ingress path: /v1/chat/completions, /v1/responses, /v1/messages.) +# +# Why a dedicated header? A proxy in front of Switchyard (e.g. LiteLLM) consumes +# the `Authorization` header for its own auth and strips it before the upstream +# call. The custom `x-switchyard-api-key` header passes through untouched. +# +# How clients send the header: +# * Codex CLI: env_http_headers / http_headers +# * OpenCode: provider options.headers +# * OpenAI / Anthropic SDK: extra / default headers +# * Claude Code: limited (can't easily send two distinct auth values) +# -- may need a wrapper/gateway unless one key serves both. +# +# Rollout order (IMPORTANT -- avoid 401s): +# 1. Deploy this build first with the route still on its current policy +# (`configured_endpoint` default, or `caller_override`) -- no behavior change. +# 2. Update clients to send `x-switchyard-api-key` (harmless under those policies). +# 3. Only after clients are confirmed sending it, switch to `caller_required`. +# +# Run: +# uv run switchyard serve \ +# --routing-profiles examples/latency_service_caller_required.yaml --port 4100 +# +# # Caller key present -> billed to that key: +# curl -s -X POST http://127.0.0.1:4100/v1/chat/completions \ +# -H 'Content-Type: application/json' \ +# -H "x-switchyard-api-key: $USER_UPSTREAM_API_KEY" \ +# -d '{"model":"my-model","messages":[{"role":"user","content":"hi"}]}' +# +# # Header omitted -> HTTP 401: +# # "missing caller API key; supply it via the x-switchyard-api-key header" +# # (the service key is never used for caller inference). +# +# Substitute your own Latency Service URL, gateway base URL, and model IDs below. +# ============================================================================= + +routes: + my-model: + type: latency_service + latency_service_url: https://your-latency-service.example.com + # Require a caller key; 401 when missing; never fall back to a service key. + credential_policy: caller_required + poll_interval_s: 10.0 + max_retries: 2 + endpoints: + # No `api_key:` on the endpoints: `caller_required` forwards the caller's + # key and never falls back, so there is no service credential to leak. + - model: primary/my-model # Latency Service endpoint_id + upstream_model: provider/my-model # model name the upstream expects + base_url: https://your-inference-gateway.example.com/v1 + # request_type: openai_responses # set for Responses-only upstreams + # (e.g. gpt-5.1-codex-mini) + - model: backup/my-model + upstream_model: provider/my-model + base_url: https://your-inference-gateway.example.com/v1 diff --git a/switchyard/lib/backends/latency_service_llm_backend.py b/switchyard/lib/backends/latency_service_llm_backend.py index 6f4d7080..77b9c444 100644 --- a/switchyard/lib/backends/latency_service_llm_backend.py +++ b/switchyard/lib/backends/latency_service_llm_backend.py @@ -140,6 +140,17 @@ def __init__( self._affinity_hits = 0 self._affinity_misses = 0 + # Per-model upstream-attempt counts for the latency-service-scoped + # /metrics breakdown, keyed by (requested model, selected upstream model, + # outcome, HTTP code). Kept separate from the layer-aggregate + # ``switchyard_upstream_attempts_total`` so that counter stays model-free. + # Cardinality is bounded: ``upstream_model`` comes from config, and + # ``requested_model`` is normalized to a configured endpoint id or the + # ``"other"`` sentinel in ``_record_model_attempt`` so a caller-supplied + # model string can't grow the series set. Mutated and read only on the + # event loop (like the affinity counters), so no lock. + self._upstream_attempts_by_model: dict[tuple[str, str, str, str], int] = {} + for ep_cfg in config.endpoints: model_id = ep_cfg.model if not model_id: @@ -345,6 +356,12 @@ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: api_key_override = self._api_key_override_for_policy( ctx.metadata.get(CTX_CALLER_API_KEY) ) + # ``caller_required`` never falls back to the configured endpoint key: + # reject before any upstream call so the service credential can't + # authenticate caller inference. ``api_key_override`` is ``None`` here + # only when no usable caller key was supplied. + if self._config.credential_policy == "caller_required" and api_key_override is None: + _reject_missing_caller_api_key(ctx) for attempt in range(1 + self._config.max_retries): # -- Route decision span: which endpoint, out of which candidates -- @@ -414,6 +431,12 @@ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: if self._stats is not None: await self._stats.record_error(model_id) outcome_metrics.record_upstream_attempt(exc.status_code) + self._record_model_attempt( + incoming_model, + upstream_model, + outcome_metrics.classify(exc.status_code), + outcome_metrics.code_label(exc.status_code), + ) # Per-event structured log (Loki) — the timestamped complement # to the aggregate outcome counter recorded above. log_upstream_attempt_failure( @@ -442,6 +465,12 @@ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: # are exactly the faults a health-aware router should absorb # by selecting a different endpoint on the next attempt. outcome_metrics.record_upstream_attempt(None) + self._record_model_attempt( + incoming_model, + upstream_model, + "retryable_error", + outcome_metrics.NO_STATUS_CODE, + ) # status_code=None → logged as code="none" (no HTTP status). log_upstream_attempt_failure( model=model_id, @@ -460,6 +489,9 @@ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: if self._stats is not None: await self._stats.record_success(model_id, backend_latency_ms) outcome_metrics.record_upstream_attempt(200) + self._record_model_attempt( + incoming_model, upstream_model, "success", "200", + ) # A successful attempt after at least one failure is direct # evidence the steering logic rescued this request. Counted # once per client request, not per recovered retry. @@ -546,12 +578,40 @@ async def _call_endpoint( ) def _api_key_override_for_policy(self, caller_api_key: object) -> str | None: - if self._config.credential_policy != "caller_override": + """Per-request key to forward upstream, or ``None`` to use the endpoint key. + + ``configured_endpoint`` never forwards a caller key. ``caller_override`` + and ``caller_required`` forward a usable caller key; they differ only in + the no-key case, which ``call`` handles before any upstream attempt + (``caller_override`` falls back to the endpoint key, ``caller_required`` + returns 401). + """ + if self._config.credential_policy == "configured_endpoint": return None if isinstance(caller_api_key, str) and caller_api_key.strip(): return caller_api_key return None + def _record_model_attempt( + self, + requested_model: str | None, + upstream_model: str, + outcome: str, + code: str, + ) -> None: + """Count one upstream attempt for the per-model /metrics breakdown. + + Event-loop only (no lock). ``requested_model`` is the client-supplied + model; it is bounded to a configured endpoint id (or the ``"other"`` + sentinel) before becoming a Prometheus label, so caller-controlled input + can't create unbounded metric-series cardinality. + """ + requested = requested_model if requested_model in self._clients else "other" + key = (requested, upstream_model, outcome, code) + self._upstream_attempts_by_model[key] = ( + self._upstream_attempts_by_model.get(key, 0) + 1 + ) + # -- Lifecycle ---------------------------------------------------------- def shutdown(self) -> None: @@ -673,6 +733,36 @@ def _render_prometheus_lines(self) -> list[str]: ) lines.append("# TYPE switchyard_affinity_misses_total counter") lines.append(f"switchyard_affinity_misses_total {self._affinity_misses}") + + # Per-model upstream-attempt breakdown — the latency-service-scoped + # complement to the layer-aggregate ``switchyard_upstream_attempts_total``. + # Series are created lazily per (requested_model, upstream_model, outcome, + # code), so the surface stays empty until traffic flows. Snapshotted to a + # plain dict for a stable view across the emission loop. + attempts_by_model = dict(self._upstream_attempts_by_model) + if attempts_by_model: + lines.append( + "# HELP switchyard_latency_upstream_attempts_total " + "Upstream call attempts from the latency-service backend, labelled " + "by requested (route) model, selected upstream model, outcome, and " + "HTTP status code (code=\"none\" for non-HTTP failures). Per-model " + "complement to the layer-aggregate switchyard_upstream_attempts_total; " + "cardinality is bounded by the configured endpoint set." + ) + lines.append("# TYPE switchyard_latency_upstream_attempts_total counter") + # Sorted for deterministic exposition order across scrapes. + for (requested_model, upstream_model, outcome, code), count in sorted( + attempts_by_model.items() + ): + labels = render_labels({ + "requested_model": requested_model, + "upstream_model": upstream_model, + "outcome": outcome, + "code": code, + }) + lines.append( + f"switchyard_latency_upstream_attempts_total{labels} {count}" + ) return lines @@ -753,6 +843,27 @@ def _stash_invalid_request_error(ctx: ProxyContext, error: Exception) -> None: } +def _reject_missing_caller_api_key(ctx: ProxyContext) -> None: + """Reject a ``caller_required`` request that carries no caller API key. + + Stashes an HTTP 401 and a provider-compatible error envelope on ``ctx`` the + same way upstream-status passthrough does, then raises so the chain errors + out before any upstream call. No upstream is contacted or counted; + ``upstream_response_from_ctx`` reads the stashed status to return a 401. This + is what keeps the configured endpoint key from ever authenticating caller + inference under the ``caller_required`` policy. + """ + ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = 401 + ctx.metadata[CTX_UPSTREAM_HTTP_BODY] = { + "error": { + "message": "missing caller API key; supply it via the x-switchyard-api-key header", + "type": "invalid_request_error", + "code": "missing_caller_api_key", + } + } + raise PermissionError("caller_required policy: missing caller API key") + + def _extract_upstream_body(error: APIStatusError) -> object | None: """Best-effort extraction of the upstream response body for passthrough. diff --git a/switchyard/lib/config/latency_service_backend_config.py b/switchyard/lib/config/latency_service_backend_config.py index 011a4059..80016d22 100644 --- a/switchyard/lib/config/latency_service_backend_config.py +++ b/switchyard/lib/config/latency_service_backend_config.py @@ -18,7 +18,9 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator LatencyServiceRequestType = Literal["openai_chat", "openai_responses"] -LatencyServiceCredentialPolicy = Literal["configured_endpoint", "caller_override"] +LatencyServiceCredentialPolicy = Literal[ + "configured_endpoint", "caller_override", "caller_required" +] class LatencyServiceEndpoint(BaseModel): @@ -81,10 +83,19 @@ class LatencyServiceBackendConfig(BaseModel): max_retries: On error, retry on a different endpoint up to this many times. Dedup prevents re-selecting an endpoint that already failed for the same request. - credential_policy: Which credential wins when the inbound HTTP - request carries a caller key. ``"configured_endpoint"`` keeps - using each endpoint's configured ``api_key``; ``"caller_override"`` - opts into BYO-key forwarding. + credential_policy: Which credential authenticates the upstream call. + The caller key is read from the ``x-switchyard-api-key`` header + (preferred — it survives proxies such as LiteLLM that strip + ``Authorization``), then ``Authorization: Bearer`` / ``x-api-key``, on + every ingress path (``/chat/completions``, ``/responses``, + ``/messages``). ``"configured_endpoint"`` always uses each endpoint's + configured ``api_key`` and ignores any caller key. ``"caller_override"`` + opts into BYO-key forwarding: a caller key is used when present, else + the call falls back to the configured ``api_key``. ``"caller_required"`` + forwards the caller key but never falls back — a request with no caller + key is rejected with HTTP 401 and the configured ``api_key`` is never + used for upstream inference (use this for per-user spend attribution, + e.g. the ``nvidia/switchyard/*`` Inference Hub routes). session_affinity: When ``True``, pin each conversation to the endpoint that first served it (cache stays warm); a pin is broken only when its endpoint degrades or the call fails. Per process. Default off. diff --git a/switchyard/lib/proxy_context.py b/switchyard/lib/proxy_context.py index ed7c2735..1248f652 100644 --- a/switchyard/lib/proxy_context.py +++ b/switchyard/lib/proxy_context.py @@ -34,8 +34,9 @@ CTX_TARGET_FORMAT = "_target_format" #: Caller-supplied API key extracted from the inbound request's -#: ``Authorization: Bearer `` (or ``x-api-key``) header. Set by the -#: HTTP endpoint after header parsing; consumed by backends that support +#: ``x-switchyard-api-key`` header (preferred — survives proxies that strip +#: ``Authorization``), or ``Authorization: Bearer `` / ``x-api-key``. Set by +#: the HTTP endpoint after header parsing; consumed by backends that support #: opt-in per-caller credential forwarding. #: Absent when the caller did not supply a credential or supplied a known #: launcher-sentinel placeholder. diff --git a/switchyard/lib/request_metadata.py b/switchyard/lib/request_metadata.py index 382f3d96..5bc28959 100644 --- a/switchyard/lib/request_metadata.py +++ b/switchyard/lib/request_metadata.py @@ -25,6 +25,21 @@ # sets ``OPENAI_API_KEY="switchyard"`` (see codex_cli_launcher.py). _CALLER_KEY_SENTINELS = frozenset({"switchyard", ""}) +# Dedicated forwarded credential header. Preferred over ``Authorization`` +# because a proxy in front of Switchyard (e.g. LiteLLM) consumes the +# ``Authorization`` header for its own auth and strips it before the upstream +# call, while a custom header passes through untouched — so a BYO-key caller +# behind such a proxy stays correctly attributed for upstream inference spend. +CALLER_API_KEY_HEADER = "x-switchyard-api-key" # pragma: allowlist secret + +# Request headers whose values carry a caller credential. Redacted before the +# header map is retained on the context (``CTX_PROFILE_REQUEST_HEADERS``), so the +# caller's key never reaches profile metadata, intake, logs, or traces. The key +# is still forwarded upstream via ``CTX_CALLER_API_KEY`` (extracted by +# ``attach_caller_api_key`` from the raw headers, before redaction). +_SENSITIVE_HEADERS = frozenset({"authorization", "x-api-key", CALLER_API_KEY_HEADER}) +_REDACTED = "[REDACTED]" + def attach_request_metadata( ctx: Any, @@ -34,10 +49,28 @@ def attach_request_metadata( """Attach request metadata to both Python and Rust-owned context storage.""" ctx.metadata[CTX_REQUEST_METADATA] = metadata if headers is not None: - ctx.metadata[CTX_PROFILE_REQUEST_HEADERS] = dict(headers) + # Redact credential headers before retaining the map: the caller key is + # already extracted into ``CTX_CALLER_API_KEY`` for upstream forwarding, + # so nothing downstream needs the raw value, and a retained/logged header + # map must not expose it. + ctx.metadata[CTX_PROFILE_REQUEST_HEADERS] = redact_sensitive_headers(headers) metadata.apply_to_context(ctx) +def redact_sensitive_headers(headers: Mapping[str, str]) -> dict[str, str]: + """Return a copy of *headers* with credential-bearing values redacted. + + Headers named in :data:`_SENSITIVE_HEADERS` (``Authorization``, ``x-api-key``, + ``x-switchyard-api-key``) have their values replaced with ``"[REDACTED]"`` so a + retained or logged header map can never expose the caller's API key. + Header-name matching is case-insensitive. + """ + return { + name: (_REDACTED if name.lower() in _SENSITIVE_HEADERS else value) + for name, value in headers.items() + } + + def attach_caller_api_key(ctx: Any, headers: Mapping[str, str]) -> None: """Attach the caller-supplied API key to *ctx* when the request carries one.""" caller_key = extract_caller_api_key(headers) @@ -48,11 +81,20 @@ def attach_caller_api_key(ctx: Any, headers: Mapping[str, str]) -> None: def extract_caller_api_key(headers: Mapping[str, str]) -> str | None: """Pull the caller-supplied API key out of an HTTP request's headers. - Precedence: ``Authorization: Bearer `` first, then ``x-api-key``. - Returns ``None`` when neither header is present, the bearer scheme is - missing, or the value is a known launcher sentinel (so coding-agent + Precedence: the dedicated ``x-switchyard-api-key`` header first, then + ``Authorization: Bearer ``, then ``x-api-key``. The dedicated header is + preferred because a proxy in front of Switchyard (e.g. LiteLLM) strips + ``Authorization`` before the upstream call; the custom header survives, so the + caller's key — not a service key — is the credential billed for upstream + inference. Returns ``None`` when no usable header is present, the bearer + scheme is missing, or the value is a known launcher sentinel (so coding-agent placeholder keys do not get forwarded upstream as real credentials). """ + forwarded = headers.get(CALLER_API_KEY_HEADER) or headers.get("X-Switchyard-Api-Key") + if forwarded: + candidate = forwarded.strip() + if candidate.lower() not in _CALLER_KEY_SENTINELS: + return candidate auth = headers.get("authorization") or headers.get("Authorization") if auth: scheme, _, value = auth.partition(" ") @@ -69,6 +111,7 @@ def extract_caller_api_key(headers: Mapping[str, str]) -> str | None: __all__ = [ + "CALLER_API_KEY_HEADER", "CTX_REQUEST_METADATA", "CTX_PROFILE_REQUEST_HEADERS", "INTAKE_APP_HEADER", @@ -80,4 +123,5 @@ def extract_caller_api_key(headers: Mapping[str, str]) -> str | None: "attach_caller_api_key", "attach_request_metadata", "extract_caller_api_key", + "redact_sensitive_headers", ] diff --git a/tests/test_latency_service_llm_backend.py b/tests/test_latency_service_llm_backend.py index 081186b0..1d058066 100644 --- a/tests/test_latency_service_llm_backend.py +++ b/tests/test_latency_service_llm_backend.py @@ -1021,6 +1021,57 @@ async def test_missing_caller_key_falls_back_to_config_for_responses(self): call_kwargs = backend._clients["model-A"].aresponses.call_args.kwargs assert call_kwargs["api_key"] is None + async def test_caller_required_policy_passes_caller_key_to_chat_sdk(self): + from switchyard.lib.proxy_context import CTX_CALLER_API_KEY + + backend = _make_backend(_config("model-A", credential_policy="caller_required")) + backend._clients["model-A"].acompletion = AsyncMock( + return_value=_make_completion() + ) + + ctx = ProxyContext() + ctx.metadata[CTX_CALLER_API_KEY] = "nvapi-caller-supplied" + await backend.call(ctx, _openai_request()) + + call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs + assert call_kwargs["api_key"] == "nvapi-caller-supplied" + + async def test_caller_required_policy_rejects_missing_caller_key(self): + """No caller key under caller_required → 401 stashed, no upstream call.""" + from switchyard.lib.proxy_context import CTX_UPSTREAM_HTTP_STATUS + + backend = _make_backend(_config("model-A", credential_policy="caller_required")) + backend._clients["model-A"].acompletion = AsyncMock( + return_value=_make_completion() + ) + + ctx = ProxyContext() + with pytest.raises(PermissionError): + await backend.call(ctx, _openai_request()) + + assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401 + backend._clients["model-A"].acompletion.assert_not_called() + + async def test_caller_required_policy_rejects_blank_caller_key(self): + """A blank caller key is unusable → 401, never the configured service key.""" + from switchyard.lib.proxy_context import ( + CTX_CALLER_API_KEY, + CTX_UPSTREAM_HTTP_STATUS, + ) + + backend = _make_backend(_config("model-A", credential_policy="caller_required")) + backend._clients["model-A"].acompletion = AsyncMock( + return_value=_make_completion() + ) + + ctx = ProxyContext() + ctx.metadata[CTX_CALLER_API_KEY] = " " + with pytest.raises(PermissionError): + await backend.call(ctx, _openai_request()) + + assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401 + backend._clients["model-A"].acompletion.assert_not_called() + @pytest.mark.parametrize( ("path", "body"), @@ -1228,6 +1279,132 @@ async def test_blank_caller_key_does_not_clobber_endpoint_key(self): assert captured["authorization"] == "Bearer ENDPOINT-CONFIGURED-KEY" + async def test_caller_required_policy_uses_caller_key(self): + """caller_required forwards the caller key to the upstream wire.""" + from switchyard.lib.proxy_context import CTX_CALLER_API_KEY + + captured: dict[str, str | None] = {} + backend = self._backend_with_captured_auth( + captured, + credential_policy="caller_required", + ) + + ctx = ProxyContext() + ctx.metadata[CTX_CALLER_API_KEY] = "CALLER-BYO-KEY" + await backend.call(ctx, _openai_request()) + + assert captured["authorization"] == "Bearer CALLER-BYO-KEY" + + async def test_caller_required_rejects_missing_key_without_upstream_call(self): + """caller_required with no caller key 401s before reaching the upstream. + + The configured ``ENDPOINT-CONFIGURED-KEY`` must never authenticate the + call — the mock transport handler should not run at all. + """ + from switchyard.lib.proxy_context import CTX_UPSTREAM_HTTP_STATUS + + captured: dict[str, str | None] = {} + backend = self._backend_with_captured_auth( + captured, + credential_policy="caller_required", + ) + + ctx = ProxyContext() + with pytest.raises(PermissionError): + await backend.call(ctx, _openai_request()) + + assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401 + assert captured == {} + + +# --------------------------------------------------------------------------- +# Per-model upstream-attempt metrics +# --------------------------------------------------------------------------- + + +class TestPerModelUpstreamMetrics: + """``switchyard_latency_upstream_attempts_total`` breaks attempts out by + requested route model, selected upstream model, outcome, and HTTP code — + the per-model complement to the layer-aggregate counter.""" + + async def test_success_attempt_recorded_per_model(self): + backend = _make_backend(_config("model-A")) + backend._clients["model-A"].acompletion = AsyncMock( + return_value=_make_completion() + ) + _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) + + # Request a configured endpoint id so ``requested_model`` is preserved. + await backend.call(ProxyContext(), _openai_request(model="model-A")) + + out = "\n".join(backend._render_prometheus_lines()) + assert ( + 'switchyard_latency_upstream_attempts_total{' + 'requested_model="model-A",upstream_model="model-A",' + 'outcome="success",code="200"} 1' + ) in out + + async def test_error_attempt_recorded_per_model(self): + backend = _make_backend(_config("model-A", max_retries=0)) + backend._clients["model-A"].acompletion = AsyncMock( + side_effect=_api_status_error(500) + ) + _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) + + with pytest.raises(openai.APIStatusError): + await backend.call(ProxyContext(), _openai_request(model="model-A")) + + out = "\n".join(backend._render_prometheus_lines()) + assert ( + 'switchyard_latency_upstream_attempts_total{' + 'requested_model="model-A",upstream_model="model-A",' + 'outcome="retryable_error",code="500"} 1' + ) in out + + async def test_unknown_requested_model_normalized_to_sentinel(self): + # A client-supplied model that is not a configured endpoint id must not + # become a raw Prometheus label (unbounded cardinality); it collapses to + # the ``"other"`` sentinel. + backend = _make_backend(_config("model-A")) + backend._clients["model-A"].acompletion = AsyncMock( + return_value=_make_completion() + ) + _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) + + await backend.call(ProxyContext(), _openai_request()) # model="incoming-model" + + out = "\n".join(backend._render_prometheus_lines()) + assert 'requested_model="other"' in out + assert 'requested_model="incoming-model"' not in out + + async def test_upstream_model_override_labels_the_upstream_name(self): + config = LatencyServiceBackendConfig( + latency_service_url=LATENCY_SERVICE_URL, + endpoints=[LatencyServiceEndpoint( + model="model-A", + base_url="http://a.test", + api_key="k", + upstream_model="azure/openai/gpt-5.1-codex-mini", + )], + ) + backend = _make_backend(config) + backend._clients["model-A"].acompletion = AsyncMock( + return_value=_make_completion() + ) + _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) + + await backend.call(ProxyContext(), _openai_request()) + + out = "\n".join(backend._render_prometheus_lines()) + assert 'upstream_model="azure/openai/gpt-5.1-codex-mini"' in out + + async def test_metric_absent_until_traffic(self): + backend = _make_backend(_config("model-A")) + + out = "\n".join(backend._render_prometheus_lines()) + + assert "switchyard_latency_upstream_attempts_total" not in out + # --------------------------------------------------------------------------- # Readiness and shutdown diff --git a/tests/test_request_metadata.py b/tests/test_request_metadata.py index bde92b43..051c931f 100644 --- a/tests/test_request_metadata.py +++ b/tests/test_request_metadata.py @@ -7,16 +7,25 @@ import pytest -from switchyard.lib.request_metadata import extract_caller_api_key +from switchyard.lib.proxy_context import CTX_CALLER_API_KEY, ProxyContext +from switchyard.lib.request_metadata import ( + CTX_PROFILE_REQUEST_HEADERS, + attach_caller_api_key, + attach_request_metadata, + extract_caller_api_key, + redact_sensitive_headers, +) +from switchyard_rust.components import RequestMetadata class TestExtractCallerApiKey: """``extract_caller_api_key`` parses caller credentials from HTTP headers. - Multi-tenant deploys forward each caller's key per request via - ``Authorization: Bearer ``; ``x-api-key`` is a documented - fallback. The codex launcher sends ``"switchyard"`` as a sentinel - placeholder, which must not be forwarded upstream. + Multi-tenant deploys forward each caller's key per request. The dedicated + ``x-switchyard-api-key`` header is preferred (it survives proxies such as + LiteLLM that strip ``Authorization``); ``Authorization: Bearer `` and + ``x-api-key`` remain supported for direct callers. The codex launcher sends + ``"switchyard"`` as a sentinel placeholder, which must not be forwarded. """ @pytest.mark.parametrize( @@ -26,6 +35,19 @@ class TestExtractCallerApiKey: ({"authorization": "bearer nvapi-lowercase"}, "nvapi-lowercase"), ({"x-api-key": "nvapi-via-x-api-key"}, "nvapi-via-x-api-key"), ({"X-Api-Key": "nvapi-titlecase-header"}, "nvapi-titlecase-header"), + # The dedicated forwarded header is honored... + ({"x-switchyard-api-key": "nvapi-forwarded"}, "nvapi-forwarded"), + ({"X-Switchyard-Api-Key": "nvapi-fwd-titlecase"}, "nvapi-fwd-titlecase"), + # ...and wins over Authorization and x-api-key when several are set + # (the case behind a proxy that strips Authorization upstream). + ( + { + "x-switchyard-api-key": "forwarded-wins", + "Authorization": "Bearer lose", + "x-api-key": "lose-too", + }, + "forwarded-wins", + ), # Authorization wins over x-api-key when both are present. ( {"Authorization": "Bearer first", "x-api-key": "second"}, @@ -33,6 +55,7 @@ class TestExtractCallerApiKey: ), # Surrounding whitespace is stripped. ({"Authorization": "Bearer nvapi-spaces "}, "nvapi-spaces"), + ({"x-switchyard-api-key": " nvapi-fwd-spaces "}, "nvapi-fwd-spaces"), ], ) def test_extraction(self, headers: dict[str, str], expected: str) -> None: @@ -46,9 +69,11 @@ def test_extraction(self, headers: dict[str, str], expected: str) -> None: {"Authorization": "Bearer "}, # Non-bearer schemes are not forwarded. {"Authorization": "Basic dXNlcjpwYXNz"}, - # Codex launcher sentinel. + # Codex launcher sentinel — in any supported header. {"Authorization": "Bearer switchyard"}, {"x-api-key": "switchyard"}, + {"x-switchyard-api-key": "switchyard"}, + {"x-switchyard-api-key": ""}, # Case-insensitive sentinel match — both halves of the case # space should be treated as the same placeholder. {"Authorization": "Bearer Switchyard"}, @@ -56,3 +81,50 @@ def test_extraction(self, headers: dict[str, str], expected: str) -> None: ) def test_no_key_returned(self, headers: dict[str, str]) -> None: assert extract_caller_api_key(headers) is None + + +class TestRedactSensitiveHeaders: + """Credential headers are scrubbed before the header map is retained.""" + + def test_redacts_credential_headers(self) -> None: + headers = { + "Authorization": "Bearer nvapi-real", + "x-api-key": "nvapi-x", + "x-switchyard-api-key": "nvapi-forwarded", + "x-switchyard-intake-app": "demo", + "content-type": "application/json", + } + redacted = redact_sensitive_headers(headers) + assert redacted["Authorization"] == "[REDACTED]" + assert redacted["x-api-key"] == "[REDACTED]" + assert redacted["x-switchyard-api-key"] == "[REDACTED]" + # Non-credential headers pass through untouched. + assert redacted["x-switchyard-intake-app"] == "demo" + assert redacted["content-type"] == "application/json" + + def test_matching_is_case_insensitive(self) -> None: + redacted = redact_sensitive_headers({"X-Switchyard-Api-Key": "nvapi-real"}) + assert redacted["X-Switchyard-Api-Key"] == "[REDACTED]" + + +class TestCallerKeyForwardedButNotRetained: + """The endpoint extracts the caller key for upstream use, then retains a + redacted header map so the key cannot leak into profile metadata, intake, + logs, or traces.""" + + def test_key_extracted_but_redacted_in_stored_headers(self) -> None: + headers = { + "x-switchyard-api-key": "nvapi-secret", + "x-switchyard-intake-app": "demo", + } + ctx = ProxyContext() + # Mirror the endpoint: both helpers receive the raw headers. + attach_request_metadata(ctx, RequestMetadata.from_headers(headers), headers) + attach_caller_api_key(ctx, headers) + + # Extracted for upstream forwarding... + assert ctx.metadata[CTX_CALLER_API_KEY] == "nvapi-secret" + # ...but the retained header map carries no raw credential. + stored = ctx.metadata[CTX_PROFILE_REQUEST_HEADERS] + assert stored["x-switchyard-api-key"] == "[REDACTED]" + assert stored["x-switchyard-intake-app"] == "demo" From dc3f1b3010181a45aeeb142c727fd4a083fecac6 Mon Sep 17 00:00:00 2001 From: eric-liu-nvidia Date: Fri, 3 Jul 2026 18:13:00 +0800 Subject: [PATCH 02/11] fix(cli): accept request_type on latency_service route-bundle endpoints --- switchyard/cli/route_bundle.py | 6 ++++-- tests/test_route_bundle.py | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py index 4ce5251b..29987864 100644 --- a/switchyard/cli/route_bundle.py +++ b/switchyard/cli/route_bundle.py @@ -200,7 +200,9 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: "timeout", "timeout_secs", }) -_LATENCY_ENDPOINT_KEYS = _LATENCY_ENDPOINT_DEFAULT_KEYS | frozenset({"model", "upstream_model"}) +_LATENCY_ENDPOINT_KEYS = _LATENCY_ENDPOINT_DEFAULT_KEYS | frozenset( + {"model", "upstream_model", "request_type"} +) _LATENCY_SERVICE_ROUTE_KEYS = _ROUTE_METADATA_KEYS | frozenset({ "defaults", "endpoints", @@ -1272,7 +1274,7 @@ def _latency_endpoint( endpoint_data = { key: data[key] - for key in ("model", "upstream_model", "api_key", "base_url", "timeout") + for key in ("model", "upstream_model", "api_key", "base_url", "timeout", "request_type") if key in data } return LatencyServiceEndpoint.model_validate(endpoint_data) diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py index a4435b7f..fef87c37 100644 --- a/tests/test_route_bundle.py +++ b/tests/test_route_bundle.py @@ -1606,6 +1606,33 @@ def test_latency_service_invalid_credential_policy_rejected_via_bundle() -> None }) +def test_latency_endpoint_request_type_reaches_backend_config() -> None: + """Endpoint-level request_type in YAML selects the upstream API surface.""" + table = build_route_bundle_table({ + "routes": { + "r": { + "type": "latency_service", + "latency_service_url": "http://ls.test:8080", + "endpoints": [ + { + "model": "codex-mini", + "api_key": "k", + "base_url": "https://ls.test/v1", + "request_type": "openai_responses", + }, + {"model": "w", "api_key": "k", "base_url": "https://ls.test/v1"}, + ], + }, + }, + }) + + backend = _latency_backend(table.lookup_switchyard("r")) + + by_model = {endpoint.model: endpoint for endpoint in backend._config.endpoints} + assert by_model["codex-mini"].request_type == "openai_responses" + assert by_model["w"].request_type == "openai_chat" + + def test_deterministic_affinity_warmup_turns_accepted_by_route_bundle() -> None: """affinity_warmup_turns is a deterministic-route knob.""" table = build_route_bundle_table({ From 43bc6b9df76d46a2d5522e890b0a4c3ba029f700 Mon Sep 17 00:00:00 2001 From: eric-liu-nvidia Date: Wed, 8 Jul 2026 01:47:53 +0800 Subject: [PATCH 03/11] fix(metrics): attribute route-model traffic in the latency upstream-attempts metric --- .agents/skills/switchyard-lib-core/SKILL.md | 2 +- switchyard/cli/route_bundle.py | 5 ++++ .../backends/latency_service_llm_backend.py | 30 ++++++++++++++----- .../config/latency_service_backend_config.py | 8 +++++ tests/test_latency_service_llm_backend.py | 29 ++++++++++++++++++ tests/test_route_bundle.py | 23 ++++++++++++++ 6 files changed, 88 insertions(+), 9 deletions(-) diff --git a/.agents/skills/switchyard-lib-core/SKILL.md b/.agents/skills/switchyard-lib-core/SKILL.md index 15d472ff..75e13e4e 100644 --- a/.agents/skills/switchyard-lib-core/SKILL.md +++ b/.agents/skills/switchyard-lib-core/SKILL.md @@ -43,7 +43,7 @@ right validation set. If the change is driven by a launcher need, also read | A fixed-path endpoint contributed by per-route components | Set `Endpoint.register_once = True`; `build_switchyard_app(...)` mounts the first instance while still running every component's lifecycle. Leave the default `False` for configurable endpoint classes that may mount distinct instances. | | Per-endpoint attribution on `/metrics` for a Python backend that can't be wrapped by `StatsLlmBackend` | Set `ctx.selected_model = endpoint_id` before returning the response. Also set `ctx.backend_call_latency_ms = upstream_call_ms` so the response processor can compute routing overhead. `LatencyServiceLLMBackend.call` is the reference. | | State metrics on `/metrics` | Register a `PrometheusEmitter` via `switchyard.lib.endpoints.prometheus_emitter.register(...)` and unregister on `shutdown()`. This is for backend-owned state, not request-flow counters. | -| Error-rate / retry-recovery counters | Use `switchyard.lib.endpoints.outcome_metrics`. FastAPI middleware records client outcomes. A retrying Python backend records each upstream attempt itself (and `record_retry_recovered()`) and sets `CTX_UPSTREAM_ATTEMPTS_RECORDED` so the endpoint skips its fallback — `LatencyServiceLLMBackend.call` is the reference. Single-attempt backends (Rust native / passthrough / multi) record nothing themselves; the endpoint fallback (`record_upstream_attempt_success` / `record_upstream_attempt_failure` in `upstream_error.py`, called from `dispatch_chat_request` and `handle_chain_exception`) counts their one attempt. Don't add a `model` label — these counters are layer-aggregate. Keep labels bounded. For a per-model error breakdown, emit a backend-owned counter via the `PrometheusEmitter` instead (bounded by the configured model set) — `LatencyServiceLLMBackend._render_prometheus_lines` exposes `switchyard_latency_upstream_attempts_total{requested_model,upstream_model,outcome,code}` next to the aggregate, leaving the shared counter model-free. | +| Error-rate / retry-recovery counters | Use `switchyard.lib.endpoints.outcome_metrics`. FastAPI middleware records client outcomes. A retrying Python backend records each upstream attempt itself (and `record_retry_recovered()`) and sets `CTX_UPSTREAM_ATTEMPTS_RECORDED` so the endpoint skips its fallback — `LatencyServiceLLMBackend.call` is the reference. Single-attempt backends (Rust native / passthrough / multi) record nothing themselves; the endpoint fallback (`record_upstream_attempt_success` / `record_upstream_attempt_failure` in `upstream_error.py`, called from `dispatch_chat_request` and `handle_chain_exception`) counts their one attempt. Don't add a `model` label — these counters are layer-aggregate. Keep labels bounded. For a per-model error breakdown, emit a backend-owned counter via the `PrometheusEmitter` instead (labels bounded to config-derived ids: the route id `config.route_model` + endpoint ids, else the `other` sentinel) — `LatencyServiceLLMBackend._render_prometheus_lines` exposes `switchyard_latency_upstream_attempts_total{requested_model,upstream_model,outcome,code}` next to the aggregate, leaving the shared counter model-free. | | Per-event error log | Use `switchyard.lib.endpoints.upstream_error_log.log_upstream_attempt_failure(...)` on the failure path. Events belong in logs/traces, not Prometheus sample timestamps. | | CLI launcher integration | Build one profile-backed `SwitchyardApp` with `build_tier_passthrough_switchyard(...)` for single-target mode, or merge route YAML with `load_route_bundle_table(...)`. Hand the result to `build_switchyard_app`. | | A new preset | Put preset helpers beside the profile config they produce, under `switchyard/lib/profiles/`. Presets should return typed config objects, not runnable chains. | diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py index 29987864..eadd6a92 100644 --- a/switchyard/cli/route_bundle.py +++ b/switchyard/cli/route_bundle.py @@ -801,6 +801,7 @@ def _build_switchyard_for_route( if route_type == "latency_service": return _latency_service_switchyard( + model_id, route, target_defaults, stats=stats, @@ -1131,6 +1132,7 @@ def _passthrough_target( def _latency_service_switchyard( + model_id: str, route: Mapping[str, object], target_defaults: Mapping[str, object], stats: StatsAccumulator, @@ -1154,6 +1156,9 @@ def _latency_service_switchyard( "latency_service.latency_service_url", ), "endpoints": endpoints, + # The YAML route key is what clients send as ``model``; hand it to the + # backend so the per-model metric can attribute route-key traffic. + "route_model": model_id, "poll_interval_s": _optional_float(route.get("poll_interval_s"), default=10.0), "poll_timeout_s": _optional_float(route.get("poll_timeout_s"), default=5.0), "max_retries": _optional_int(route.get("max_retries"), default=2), diff --git a/switchyard/lib/backends/latency_service_llm_backend.py b/switchyard/lib/backends/latency_service_llm_backend.py index 77b9c444..1d1d3da8 100644 --- a/switchyard/lib/backends/latency_service_llm_backend.py +++ b/switchyard/lib/backends/latency_service_llm_backend.py @@ -145,10 +145,11 @@ def __init__( # outcome, HTTP code). Kept separate from the layer-aggregate # ``switchyard_upstream_attempts_total`` so that counter stays model-free. # Cardinality is bounded: ``upstream_model`` comes from config, and - # ``requested_model`` is normalized to a configured endpoint id or the - # ``"other"`` sentinel in ``_record_model_attempt`` so a caller-supplied - # model string can't grow the series set. Mutated and read only on the - # event loop (like the affinity counters), so no lock. + # ``requested_model`` is normalized to a config-derived id (route id or + # endpoint id) or the ``"other"`` sentinel in ``_record_model_attempt`` + # so a caller-supplied model string can't grow the series set. Mutated + # and read only on the event loop (like the affinity counters), so no + # lock. self._upstream_attempts_by_model: dict[tuple[str, str, str, str], int] = {} for ep_cfg in config.endpoints: @@ -190,6 +191,15 @@ def __init__( ep_cfg.base_url, ) + # Bounded ``requested_model`` label set: the configured endpoint ids + # plus the client-facing route id when the deployment supplied one + # (IH clients request the route key, e.g. ``nvidia/switchyard/gpt-5.4``, + # never an endpoint id). Both sources are config-derived, so metric + # cardinality stays bounded. + self._requested_model_ids = frozenset(self._clients) | ( + frozenset((config.route_model,)) if config.route_model else frozenset() + ) + self._poller = HealthPoller( latency_service_url=config.latency_service_url, model_ids=list(self._clients.keys()), @@ -602,11 +612,15 @@ def _record_model_attempt( """Count one upstream attempt for the per-model /metrics breakdown. Event-loop only (no lock). ``requested_model`` is the client-supplied - model; it is bounded to a configured endpoint id (or the ``"other"`` - sentinel) before becoming a Prometheus label, so caller-controlled input - can't create unbounded metric-series cardinality. + model; it is bounded to a config-derived id — the route id + (``config.route_model``) or a configured endpoint id — with the + ``"other"`` sentinel as fallback before becoming a Prometheus label, so + caller-controlled input can't create unbounded metric-series + cardinality. """ - requested = requested_model if requested_model in self._clients else "other" + requested = ( + requested_model if requested_model in self._requested_model_ids else "other" + ) key = (requested, upstream_model, outcome, code) self._upstream_attempts_by_model[key] = ( self._upstream_attempts_by_model.get(key, 0) + 1 diff --git a/switchyard/lib/config/latency_service_backend_config.py b/switchyard/lib/config/latency_service_backend_config.py index 80016d22..dd27b883 100644 --- a/switchyard/lib/config/latency_service_backend_config.py +++ b/switchyard/lib/config/latency_service_backend_config.py @@ -83,6 +83,13 @@ class LatencyServiceBackendConfig(BaseModel): max_retries: On error, retry on a different endpoint up to this many times. Dedup prevents re-selecting an endpoint that already failed for the same request. + route_model: Client-facing route id this backend serves (the + route-table / YAML route key, e.g. + ``"nvidia/switchyard/gpt-5.4"``). Metrics-only: it joins the + bounded ``requested_model`` label set on + ``switchyard_latency_upstream_attempts_total`` so route-key + traffic is attributed instead of collapsing to ``"other"``. + Has no effect on routing. credential_policy: Which credential authenticates the upstream call. The caller key is read from the ``x-switchyard-api-key`` header (preferred — it survives proxies such as LiteLLM that strip @@ -114,6 +121,7 @@ class LatencyServiceBackendConfig(BaseModel): latency_service_url: str = "" endpoints: list[LatencyServiceEndpoint] = Field(default_factory=list) + route_model: str | None = None poll_interval_s: float = 10.0 poll_timeout_s: float = 5.0 max_retries: int = 2 diff --git a/tests/test_latency_service_llm_backend.py b/tests/test_latency_service_llm_backend.py index 1d058066..53657eb9 100644 --- a/tests/test_latency_service_llm_backend.py +++ b/tests/test_latency_service_llm_backend.py @@ -1361,6 +1361,35 @@ async def test_error_attempt_recorded_per_model(self): 'outcome="retryable_error",code="500"} 1' ) in out + async def test_route_model_preserved_as_requested_model_label(self): + # Clients of a deployed route send the route key (e.g. + # ``nvidia/switchyard/gpt-5.4``), never an endpoint id. The route id is + # config-derived, so it joins the bounded label set instead of + # collapsing to the ``"other"`` sentinel. + config = LatencyServiceBackendConfig( + latency_service_url=LATENCY_SERVICE_URL, + endpoints=[_ep("azure/openai/gpt-5.4")], + route_model="nvidia/switchyard/gpt-5.4", + ) + backend = _make_backend(config) + backend._clients["azure/openai/gpt-5.4"].acompletion = AsyncMock( + return_value=_make_completion() + ) + _set_health(backend, {"azure/openai/gpt-5.4": EndpointHealthStatus.HEALTHY}) + + await backend.call( + ProxyContext(), _openai_request(model="nvidia/switchyard/gpt-5.4") + ) + + out = "\n".join(backend._render_prometheus_lines()) + assert ( + 'switchyard_latency_upstream_attempts_total{' + 'requested_model="nvidia/switchyard/gpt-5.4",' + 'upstream_model="azure/openai/gpt-5.4",' + 'outcome="success",code="200"} 1' + ) in out + assert 'requested_model="other"' not in out + async def test_unknown_requested_model_normalized_to_sentinel(self): # A client-supplied model that is not a configured endpoint id must not # become a raw Prometheus label (unbounded cardinality); it collapses to diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py index fef87c37..4bbc7bd0 100644 --- a/tests/test_route_bundle.py +++ b/tests/test_route_bundle.py @@ -1633,6 +1633,29 @@ def test_latency_endpoint_request_type_reaches_backend_config() -> None: assert by_model["w"].request_type == "openai_chat" +def test_latency_route_key_reaches_backend_as_route_model() -> None: + """The YAML route key becomes the backend's metrics route_model id.""" + table = build_route_bundle_table({ + "routes": { + "nvidia/switchyard/gpt-5.4": { + "type": "latency_service", + "latency_service_url": "http://ls.test:8080", + "endpoints": [ + { + "model": "azure/openai/gpt-5.4", + "api_key": "k", + "base_url": "https://ls.test/v1", + }, + ], + }, + }, + }) + + backend = _latency_backend(table.lookup_switchyard("nvidia/switchyard/gpt-5.4")) + + assert backend._config.route_model == "nvidia/switchyard/gpt-5.4" + + def test_deterministic_affinity_warmup_turns_accepted_by_route_bundle() -> None: """affinity_warmup_turns is a deterministic-route knob.""" table = build_route_bundle_table({ From 84684cac58cc0546393584955402d12f653b5507 Mon Sep 17 00:00:00 2001 From: eric-liu-nvidia Date: Wed, 8 Jul 2026 02:02:20 +0800 Subject: [PATCH 04/11] feat(latency-service): error-source annotation + exact Responses passthrough, body and stream --- .../switchyard-components/src/stats/usage.rs | 44 ++- .../tests/stats_processors.rs | 41 +++ .../backends/latency_service_llm_backend.py | 36 +- switchyard/lib/endpoints/error_envelope.py | 45 ++- switchyard/lib/endpoints/upstream_error.py | 62 +++- .../lib/endpoints/upstream_error_log.py | 11 +- switchyard/lib/llm_client.py | 68 +++- ...stats_response_processor_live_collector.py | 9 +- switchyard/lib/proxy_context.py | 27 ++ switchyard_rust/translation.py | 49 ++- tests/test_error_source_annotation.py | 156 ++++++++ tests/test_latency_service_llm_backend.py | 108 +++++- tests/test_latency_service_spans.py | 6 + tests/test_live_stats_collector.py | 33 ++ tests/test_llm_client.py | 128 ++++++- tests/test_response_translation_engine.py | 72 ++++ tests/test_upstream_error_log.py | 5 + tests/test_upstream_error_passthrough.py | 337 ++++++++++++++++++ 18 files changed, 1216 insertions(+), 21 deletions(-) create mode 100644 tests/test_error_source_annotation.py diff --git a/crates/switchyard-components/src/stats/usage.rs b/crates/switchyard-components/src/stats/usage.rs index 4411307f..884b80c9 100644 --- a/crates/switchyard-components/src/stats/usage.rs +++ b/crates/switchyard-components/src/stats/usage.rs @@ -47,16 +47,54 @@ pub fn openai_chat_usage_from_stream_event(event: &StreamEvent) -> Option Option { - let StreamEvent::Json(value) = event else { - return None; - }; + match event { + StreamEvent::Json(value) => usage_from_responses_value(value), + StreamEvent::Text(text) => sse_data_payloads(text) + .iter() + .find_map(usage_from_responses_value), + } +} + +/// Reads `response.usage` from one decoded Responses stream event. +fn usage_from_responses_value(value: &Value) -> Option { value .get("response") .and_then(|response| response.get("usage")) .and_then(usage_from_candidate) } +/// Parses the JSON `data:` payload(s) out of a raw SSE frame string. +/// +/// Per the SSE contract, a frame's `data:` lines join with newlines to form +/// one payload; a single leading space after the colon is stripped. Comment +/// frames, `[DONE]` sentinels, and non-JSON payloads yield nothing. +fn sse_data_payloads(text: &str) -> Vec { + let mut payloads = Vec::new(); + for block in text.split("\n\n") { + let data_lines: Vec<&str> = block + .split('\n') + .filter_map(|line| line.strip_prefix("data:")) + .map(|value| value.strip_prefix(' ').unwrap_or(value)) + .collect(); + if data_lines.is_empty() { + continue; + } + let data = data_lines.join("\n"); + if data.trim() == "[DONE]" { + continue; + } + if let Ok(parsed) = serde_json::from_str::(&data) { + payloads.push(parsed); + } + } + payloads +} + /// Accumulates Anthropic streaming usage and commits once at `message_stop`. #[derive(Clone, Copy, Debug, Default)] pub struct AnthropicStreamUsage { diff --git a/crates/switchyard-components/tests/stats_processors.rs b/crates/switchyard-components/tests/stats_processors.rs index a9e90ad5..3fdea5d9 100644 --- a/crates/switchyard-components/tests/stats_processors.rs +++ b/crates/switchyard-components/tests/stats_processors.rs @@ -448,6 +448,47 @@ async fn streaming_response_is_forwarded_and_records_nested_responses_usage() -> Ok(()) } +// Raw SSE frame strings (verbatim Responses passthrough) still record usage. +#[tokio::test] +async fn raw_sse_frame_responses_stream_passes_through_and_records_usage() -> Result<()> { + let accumulator = StatsAccumulator::new(); + let processor = StatsResponseProcessor::new(accumulator.clone()); + let mut ctx = ProxyContext::new(); + record_backend_selection(&mut ctx, ModelId::new("raw-frame-model")?); + ctx.insert(StatsRequestStart::now()); + + let events = vec![ + StreamEvent::Text( + "event: response.output_text.delta\n\ + data: {\"type\":\"response.output_text.delta\",\"delta\":\"hi\"}\n\n" + .to_string(), + ), + StreamEvent::Text(": keep-alive\n\n".to_string()), + StreamEvent::Text( + "event: response.completed\n\ + data: {\"type\":\"response.completed\",\"response\":{\"usage\":\ + {\"input_tokens\":8,\"output_tokens\":13,\ + \"input_tokens_details\":{\"cached_tokens\":5}}}}\n\n" + .to_string(), + ), + ]; + let stream_events = events.clone(); + let stream = futures_util::stream::iter(stream_events.into_iter().map(Ok)); + let response = ChatResponse::OpenAiResponsesStream(Box::pin(stream)); + + let processed = processor.process(&mut ctx, response).await?; + let drained = drain_responses_stream(processed).await?; + + // Frames pass through byte-identical; usage still lands in the snapshot. + assert_eq!(drained, events); + let snapshot = accumulator.snapshot()?; + let model = model_stats(&snapshot, "raw-frame-model")?; + assert_eq!(model.prompt_tokens, 8); + assert_eq!(model.completion_tokens, 13); + assert_eq!(model.cached_tokens, 5); + Ok(()) +} + // OpenAI streams should record the first real usage block only. #[tokio::test] async fn openai_chat_stream_records_first_usage_chunk_only_with_details() -> Result<()> { diff --git a/switchyard/lib/backends/latency_service_llm_backend.py b/switchyard/lib/backends/latency_service_llm_backend.py index 1d1d3da8..81983aac 100644 --- a/switchyard/lib/backends/latency_service_llm_backend.py +++ b/switchyard/lib/backends/latency_service_llm_backend.py @@ -44,9 +44,13 @@ from switchyard.lib.prometheus_exposition import format_number, render_labels from switchyard.lib.proxy_context import ( CTX_CALLER_API_KEY, + CTX_ERROR_SOURCE, CTX_UPSTREAM_ATTEMPTS_RECORDED, CTX_UPSTREAM_HTTP_BODY, CTX_UPSTREAM_HTTP_STATUS, + CTX_UPSTREAM_MODEL, + ERROR_SOURCE_PROVIDER, + ERROR_SOURCE_SWITCHYARD, ProxyContext, ) from switchyard.lib.roles import LLMBackend @@ -362,6 +366,10 @@ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: pinned_endpoint = self._affinity.pinned(ctx, request) last_exc: Exception | None = None + # Upstream model of the last failed attempt, surfaced on the error + # envelope/span/log so operators can tell *which* endpoint the + # provider-originated failure came from. + last_upstream_model: str | None = None tried: set[str] = set() api_key_override = self._api_key_override_for_policy( ctx.metadata.get(CTX_CALLER_API_KEY) @@ -437,6 +445,8 @@ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: "switchyard.outcome": outcome_metrics.classify(exc.status_code), "switchyard.upstream_status_code": exc.status_code, "switchyard.error_code": outcome_metrics.code_label(exc.status_code), + "switchyard.error_source": ERROR_SOURCE_PROVIDER, + "switchyard.upstream_model": upstream_model, }) if self._stats is not None: await self._stats.record_error(model_id) @@ -454,8 +464,10 @@ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: attempt=attempt + 1, status_code=exc.status_code, error=exc, + upstream_model=upstream_model, ) last_exc = exc + last_upstream_model = upstream_model # A client error (4xx other than 408/429) is deterministic — # replicas reject the same payload identically — so fail fast # instead of burning attempts; the post-loop passthrough @@ -467,6 +479,8 @@ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: set_tags(attempt_span, { "switchyard.outcome": "retryable_error", "switchyard.error_code": outcome_metrics.NO_STATUS_CODE, + "switchyard.error_source": ERROR_SOURCE_PROVIDER, + "switchyard.upstream_model": upstream_model, }) if self._stats is not None: await self._stats.record_error(model_id) @@ -487,8 +501,10 @@ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: attempt=attempt + 1, status_code=None, error=exc, + upstream_model=upstream_model, ) last_exc = exc + last_upstream_model = upstream_model continue backend_latency_ms = (time.monotonic() - started_at) * 1000.0 @@ -541,6 +557,15 @@ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: if upstream_body is not None: ctx.metadata[CTX_UPSTREAM_HTTP_BODY] = upstream_body + # Failure-origin annotation for the client-facing error envelope: + # every exhausted attempt failed at a selected upstream, so what the + # endpoint surfaces — the HTTP passthrough above, or the 500 for + # status-less network failures — is provider-originated. (The stash + # helpers below mark Switchyard-originated rejections instead.) + ctx.metadata[CTX_ERROR_SOURCE] = ERROR_SOURCE_PROVIDER + if last_upstream_model is not None: + ctx.metadata[CTX_UPSTREAM_MODEL] = last_upstream_model + raise last_exc # type: ignore[misc] def _body_for_endpoint_request_type( @@ -791,7 +816,10 @@ def _chat_response_for_request_type( result: object, ) -> ChatResponse: if request_type == ChatRequestType.OPENAI_RESPONSES: - if isinstance(result, AsyncStream): + # Streaming yields raw SSE frame strings (``RawSSEFrameStream``) so the + # upstream events pass through verbatim; anything async-iterable is a + # stream, a plain mapping is the exact non-streaming JSON body. + if isinstance(result, AsyncStream) or hasattr(result, "__aiter__"): return ChatResponse.openai_responses_stream(ResponsesApiStream(result)) return ChatResponse.openai_responses_completion(result) if isinstance(result, AsyncStream): @@ -855,6 +883,9 @@ def _stash_invalid_request_error(ctx: ProxyContext, error: Exception) -> None: "code": "invalid_value", } } + # This 400 rides the upstream-status channel but is Switchyard's own + # translation rejection — label it so the error-source header is truthful. + ctx.metadata[CTX_ERROR_SOURCE] = ERROR_SOURCE_SWITCHYARD def _reject_missing_caller_api_key(ctx: ProxyContext) -> None: @@ -875,6 +906,9 @@ def _reject_missing_caller_api_key(ctx: ProxyContext) -> None: "code": "missing_caller_api_key", } } + # Rides the upstream-status channel but no upstream was contacted — this + # is Switchyard's own credential-policy rejection. + ctx.metadata[CTX_ERROR_SOURCE] = ERROR_SOURCE_SWITCHYARD raise PermissionError("caller_required policy: missing caller API key") diff --git a/switchyard/lib/endpoints/error_envelope.py b/switchyard/lib/endpoints/error_envelope.py index 5d02efce..d4e99533 100644 --- a/switchyard/lib/endpoints/error_envelope.py +++ b/switchyard/lib/endpoints/error_envelope.py @@ -10,8 +10,23 @@ from fastapi.responses import JSONResponse +from switchyard.lib.proxy_context import ERROR_SOURCE_PROVIDER, ERROR_SOURCE_SWITCHYARD + _DEFAULT_UPSTREAM_MESSAGE = "upstream returned HTTP {status}" +#: Response header naming the layer that originated the error: ``switchyard`` +#: (this proxy rejected or failed the request itself) or ``provider`` (an +#: upstream LLM failure passed through). Layers above Switchyard (e.g. a +#: LiteLLM front proxy) are expected to tag their own failures the same way +#: and propagate this header from below — Switchyard cannot see them. The +#: values live in :mod:`switchyard.lib.proxy_context` so FastAPI-free backend +#: code can stamp them. +ERROR_SOURCE_HEADER = "x-switchyard-error-source" + +#: Response header carrying the upstream model actually attempted when the +#: surfaced failure happened, when a routing selection took place. +UPSTREAM_MODEL_HEADER = "x-switchyard-upstream-model" + def error_payload( message: str, @@ -38,19 +53,43 @@ def error_response( error_type: str, code: str, extra: Mapping[str, object] | None = None, + error_source: str | None = ERROR_SOURCE_SWITCHYARD, + upstream_model: str | None = None, ) -> JSONResponse: - """Build a JSONResponse with Switchyard's normalized error envelope.""" + """Build a JSONResponse with Switchyard's normalized error envelope. + + Stamps the failure-source headers: every direct caller synthesizes a + Switchyard-originated envelope, so ``error_source`` defaults to + ``switchyard``; the upstream passthrough path overrides it with + ``provider``. Headers rather than body fields keep the passthrough + contract intact — provider error bodies flow through unmodified. + """ + headers: dict[str, str] = {} + if error_source: + headers[ERROR_SOURCE_HEADER] = error_source + if upstream_model: + headers[UPSTREAM_MODEL_HEADER] = upstream_model return JSONResponse( status_code=status_code, content=error_payload(message, error_type=error_type, code=code, extra=extra), + headers=headers or None, ) def upstream_error_response( status_code: int, body: object, + *, + error_source: str = ERROR_SOURCE_PROVIDER, + upstream_model: str | None = None, ) -> JSONResponse: - """Normalize an upstream provider error body into Switchyard's envelope.""" + """Normalize an upstream provider error body into Switchyard's envelope. + + ``error_source`` defaults to ``provider`` — this path renders upstream + failures — but a backend that deliberately routes its own rejection + through the upstream-status stash (e.g. the ``caller_required`` 401) + overrides it back to ``switchyard`` via ``ctx``. + """ parsed = _upstream_error_fields(status_code, body) return error_response( status_code, @@ -58,6 +97,8 @@ def upstream_error_response( error_type=parsed.error_type, code=parsed.code, extra=parsed.extra, + error_source=error_source, + upstream_model=upstream_model, ) diff --git a/switchyard/lib/endpoints/upstream_error.py b/switchyard/lib/endpoints/upstream_error.py index 895f4357..f6015a3c 100644 --- a/switchyard/lib/endpoints/upstream_error.py +++ b/switchyard/lib/endpoints/upstream_error.py @@ -19,11 +19,19 @@ from fastapi.responses import JSONResponse from switchyard.lib.endpoints import outcome_metrics -from switchyard.lib.endpoints.error_envelope import error_response, upstream_error_response +from switchyard.lib.endpoints.error_envelope import ( + ERROR_SOURCE_HEADER, + error_response, + upstream_error_response, +) from switchyard.lib.proxy_context import ( + CTX_ERROR_SOURCE, CTX_UPSTREAM_ATTEMPTS_RECORDED, CTX_UPSTREAM_HTTP_BODY, CTX_UPSTREAM_HTTP_STATUS, + CTX_UPSTREAM_MODEL, + ERROR_SOURCE_PROVIDER, + ERROR_SOURCE_SWITCHYARD, ) from switchyard_rust.core import SwitchyardUpstreamError @@ -97,10 +105,31 @@ def upstream_response_from_ctx( """ status = ctx.metadata.get(CTX_UPSTREAM_HTTP_STATUS) if isinstance(status, int): - return upstream_error_response(status, ctx.metadata.get(CTX_UPSTREAM_HTTP_BODY)) + return upstream_error_response( + status, + ctx.metadata.get(CTX_UPSTREAM_HTTP_BODY), + # A stashed status without an explicit source is an upstream + # passthrough; backends that reuse this channel for their own + # rejections (caller_required 401, translation 400) mark the + # stash ``switchyard`` so the header stays truthful. + error_source=_ctx_error_source(ctx, default=ERROR_SOURCE_PROVIDER), + upstream_model=_ctx_upstream_model(ctx), + ) return upstream_response_from_error(exc, inbound=inbound) +def _ctx_error_source(ctx: ProxyContext, *, default: str) -> str: + """Failure origin stamped by the backend, or ``default`` for the path.""" + source = ctx.metadata.get(CTX_ERROR_SOURCE) + return source if isinstance(source, str) and source else default + + +def _ctx_upstream_model(ctx: ProxyContext) -> str | None: + """Upstream model recorded at failure time, when a selection happened.""" + model = ctx.metadata.get(CTX_UPSTREAM_MODEL) + return model if isinstance(model, str) and model else None + + def upstream_response_from_error( exc: BaseException | None, *, @@ -125,7 +154,13 @@ def _parse_body(raw: str) -> object: return text -def internal_chain_error_response(exc: BaseException, inbound: Inbound) -> JSONResponse: +def internal_chain_error_response( + exc: BaseException, + inbound: Inbound, + *, + error_source: str = ERROR_SOURCE_SWITCHYARD, + upstream_model: str | None = None, +) -> JSONResponse: """Translate an unexpected chain failure into the client error envelope. Used when an exception escapes dispatch or response-processing that is not @@ -135,6 +170,11 @@ def internal_chain_error_response(exc: BaseException, inbound: Inbound) -> JSONR this helper so the full context is preserved server-side. ``inbound`` is retained in the signature because endpoint callers already pass it, but the HTTP envelope is intentionally shared across inbound formats. + + ``error_source`` defaults to ``switchyard`` (an unexpected internal + failure) but a backend that failed on a status-less upstream fault (e.g. + a network error after retries) marks ``ctx`` so this 500 is labeled + ``provider`` instead. """ message = repr(exc)[:200] return error_response( @@ -142,6 +182,8 @@ def internal_chain_error_response(exc: BaseException, inbound: Inbound) -> JSONR message, error_type="internal_error", code="internal_chain_error", + error_source=error_source, + upstream_model=upstream_model, ) @@ -158,7 +200,12 @@ def handle_chain_exception( if upstream is not None: return upstream _log.error(log_msg, exc_info=exc) - return internal_chain_error_response(exc, inbound=inbound) + return internal_chain_error_response( + exc, + inbound=inbound, + error_source=_ctx_error_source(ctx, default=ERROR_SOURCE_SWITCHYARD), + upstream_model=_ctx_upstream_model(ctx), + ) def context_exhausted_response(exc: BaseException, inbound: Inbound) -> JSONResponse: @@ -173,4 +220,9 @@ def context_exhausted_response(exc: BaseException, inbound: Inbound) -> JSONResp "message": str(exc), "code": "context_length_exceeded", } - return JSONResponse(status_code=400, content={"error": error}) + return JSONResponse( + status_code=400, + content={"error": error}, + # Chain-executor rejection, not an upstream failure. + headers={ERROR_SOURCE_HEADER: ERROR_SOURCE_SWITCHYARD}, + ) diff --git a/switchyard/lib/endpoints/upstream_error_log.py b/switchyard/lib/endpoints/upstream_error_log.py index 6cd235e7..70977506 100644 --- a/switchyard/lib/endpoints/upstream_error_log.py +++ b/switchyard/lib/endpoints/upstream_error_log.py @@ -50,27 +50,36 @@ def log_upstream_attempt_failure( attempt: int, status_code: int | None, error: BaseException, + upstream_model: str | None = None, ) -> None: """Emit one structured JSON record for a single failed upstream attempt. ``status_code`` is the raw upstream HTTP status, or ``None`` for a non-HTTP failure (network error, pre-status timeout) — recorded as ``status_code: null`` with ``code="none"``. ``attempt`` is 1-based. + ``upstream_model`` is the model actually sent upstream (``body["model"]``) + when the caller knows it; ``model`` remains the internal route/endpoint id. ``code`` and ``outcome`` mirror the labels on ``switchyard_upstream_attempts_total`` so the event log is joinable to - the aggregate counter. The record is logged at WARNING. + the aggregate counter. ``error_source`` is always ``provider`` — this + event exists only for failures of actual upstream attempts. The record + is logged at WARNING. """ record = { "event": EVENT_NAME, "timestamp": datetime.now(UTC).isoformat(), "model": model, + "upstream_model": upstream_model, "attempt": attempt, "status_code": status_code, "code": code_label(status_code), # None (non-HTTP failure) is a retryable_error, matching how # record_upstream_attempt buckets it. "outcome": "retryable_error" if status_code is None else classify(status_code), + # Attempt failures are upstream-side by definition; the constant field + # keeps the event joinable to the response-header/span vocabulary. + "error_source": "provider", "error_type": type(error).__name__, "error": str(error)[:_MAX_ERROR_CHARS], } diff --git a/switchyard/lib/llm_client.py b/switchyard/lib/llm_client.py index fdf3f75d..c01a4e72 100644 --- a/switchyard/lib/llm_client.py +++ b/switchyard/lib/llm_client.py @@ -120,5 +120,71 @@ async def aresponses( ``api_key`` is ``None`` or blank, no override is applied and the construction-time key configured on the client is used. SDK validation and upstream errors are intentionally propagated unchanged. + + Non-streaming calls return the upstream's **exact JSON body** (a + ``dict``) rather than the SDK's typed ``Response`` model: round-tripping + through the typed model re-normalizes the payload and its + ``exclude_none`` serialization drops explicit-null fields, eroding + schema fidelity for Responses passthrough. + + Streaming calls return a :class:`RawSSEFrameStream` yielding the + upstream's SSE frames as **verbatim strings** (modulo CRLF → LF line + normalization) for the same reason — the SDK's typed event stream + drops provider extras and explicit nulls per event. The HTTP request + is sent (and error statuses raise) *before* this method returns, so + the caller's retry/failover contract is unchanged. """ - return await self._client_for_api_key(api_key).responses.create(**kwargs) + client = self._client_for_api_key(api_key) + if kwargs.get("stream"): + cm = client.responses.with_streaming_response.create(**kwargs) + # Enter eagerly: the request goes out and non-2xx statuses raise + # ``APIStatusError`` here, not at first iteration — after first + # iteration the endpoint has already committed an HTTP 200. + response = await cm.__aenter__() + return RawSSEFrameStream(cm, response.http_response) + raw = await client.responses.with_raw_response.create(**kwargs) + return raw.http_response.json() + + +class RawSSEFrameStream: + """Async iterator over an SSE response's frames as verbatim strings. + + Each item is one complete frame (all lines up to and including the blank + separator, e.g. ``"event: x\\ndata: {...}\\n\\n"``), byte-equivalent to the + upstream modulo CRLF → LF normalization. Comment/keep-alive frames pass + through unchanged. ``aclose`` releases the underlying HTTP response and is + safe to call at any point, including before the first ``__anext__``. + """ + + def __init__(self, cm: Any, http_response: Any) -> None: + self._cm = cm + self._lines = http_response.aiter_lines() + self._closed = False + + def __aiter__(self) -> RawSSEFrameStream: + return self + + async def __anext__(self) -> str: + buffer: list[str] = [] + try: + async for line in self._lines: + if line == "": + if buffer: + return "\n".join(buffer) + "\n\n" + continue + buffer.append(line) + except BaseException: + await self.aclose() + raise + await self.aclose() + if buffer: + # Upstream closed without a trailing blank line; emit the tail as + # a well-formed frame rather than dropping it. + return "\n".join(buffer) + "\n\n" + raise StopAsyncIteration + + async def aclose(self) -> None: + if self._closed: + return + self._closed = True + await self._cm.__aexit__(None, None, None) diff --git a/switchyard/lib/processors/stats_response_processor_live_collector.py b/switchyard/lib/processors/stats_response_processor_live_collector.py index fe5da193..2f140f53 100644 --- a/switchyard/lib/processors/stats_response_processor_live_collector.py +++ b/switchyard/lib/processors/stats_response_processor_live_collector.py @@ -32,6 +32,7 @@ ChatResponseType, response_type_matches, ) +from switchyard_rust.translation import sse_frame_payloads if TYPE_CHECKING: from switchyard.lib.endpoints.base import Endpoint @@ -293,7 +294,13 @@ async def _tap(event) -> None: # type: ignore[no-untyped-def] nonlocal seen if seen: return - inner = _field(event, "response") + # Verbatim-passthrough backends yield raw SSE frame strings; parse the + # frame's data payload(s) to find the usage-carrying event. + candidates = sse_frame_payloads(event) if isinstance(event, str) else [event] + inner = next( + (found for c in candidates if (found := _field(c, "response")) is not None), + None, + ) if inner is None: return u = _field(inner, "usage") diff --git a/switchyard/lib/proxy_context.py b/switchyard/lib/proxy_context.py index 1248f652..34bd5c29 100644 --- a/switchyard/lib/proxy_context.py +++ b/switchyard/lib/proxy_context.py @@ -66,9 +66,33 @@ #: loop — those rely on the endpoint fallback. CTX_UPSTREAM_ATTEMPTS_RECORDED = "_upstream_attempts_recorded" +#: Which layer originated the failure being surfaced to the client: +#: ``"provider"`` for an upstream LLM failure passed through, ``"switchyard"`` +#: for an error this proxy synthesized (credential rejection, translation +#: rejection, routing failure). Written by backends on the error path; read by +#: the endpoint layer to stamp the ``x-switchyard-error-source`` response +#: header. Unset means the endpoint's per-path default applies (``provider`` +#: for a stashed upstream status, ``switchyard`` for synthesized envelopes). +CTX_ERROR_SOURCE = "_error_source" + +#: :data:`CTX_ERROR_SOURCE` value for errors Switchyard itself originated. +#: Defined here (not in the endpoints layer) so backends can stamp it without +#: importing FastAPI-dependent modules. +ERROR_SOURCE_SWITCHYARD = "switchyard" + +#: :data:`CTX_ERROR_SOURCE` value for upstream provider failures passed through. +ERROR_SOURCE_PROVIDER = "provider" + +#: Upstream model actually attempted when the surfaced failure happened, when +#: a routing selection took place. Written alongside +#: :data:`CTX_ERROR_SOURCE`; read by the endpoint layer to stamp the +#: ``x-switchyard-upstream-model`` response header. +CTX_UPSTREAM_MODEL = "_upstream_model" + __all__ = [ "CTX_CALLER_API_KEY", + "CTX_ERROR_SOURCE", "CTX_ORIGINAL_FORMAT", "CTX_ORIGINAL_MODEL", "CTX_ORIGINAL_REQUEST", @@ -78,5 +102,8 @@ "CTX_UPSTREAM_ATTEMPTS_RECORDED", "CTX_UPSTREAM_HTTP_BODY", "CTX_UPSTREAM_HTTP_STATUS", + "CTX_UPSTREAM_MODEL", + "ERROR_SOURCE_PROVIDER", + "ERROR_SOURCE_SWITCHYARD", "ProxyContext", ] diff --git a/switchyard_rust/translation.py b/switchyard_rust/translation.py index 0b6a964a..07259315 100644 --- a/switchyard_rust/translation.py +++ b/switchyard_rust/translation.py @@ -231,8 +231,9 @@ async def translate_stream( ) try: async for event in stream: - for translated in translator.translate_event(_jsonable_mapping(event)): - yield _coerce_stream_output(target_format, translated, output) + for payload in _stream_event_payloads(event): + for translated in translator.translate_event(payload): + yield _coerce_stream_output(target_format, translated, output) for translated in translator.finish(): yield _coerce_stream_output(target_format, translated, output) finally: @@ -426,6 +427,50 @@ def _jsonable_mapping(value: Any) -> dict[str, Any]: return dict(value) if isinstance(value, Mapping) else {} +def _stream_event_payloads(event: Any) -> list[dict[str, Any]]: + """JSON payload(s) of one stream item, for cross-format translation. + + Backends that preserve wire fidelity yield raw SSE frame *strings* + (see ``RawSSEFrameStream``); those are parsed here so the translator + still sees event dicts. Every other item shape keeps the existing + jsonable-mapping coercion. + """ + if isinstance(event, str): + return sse_frame_payloads(event) + return [_jsonable_mapping(event)] + + +def sse_frame_payloads(frame: str) -> list[dict[str, Any]]: + """Parse an SSE frame string into its JSON ``data:`` payload(s). + + Follows the SSE contract: a frame's ``data:`` lines are joined with + newlines to form one payload; an optional single leading space after the + colon is stripped. Comment/keep-alive frames, the ``[DONE]`` sentinel, + and non-JSON or non-object payloads yield nothing. Accepts a string + containing multiple ``\\n\\n``-separated frames and returns payloads in + order. + """ + payloads: list[dict[str, Any]] = [] + for block in frame.split("\n\n"): + data_lines: list[str] = [] + for line in block.split("\n"): + if line.startswith("data:"): + value = line[5:] + data_lines.append(value[1:] if value.startswith(" ") else value) + if not data_lines: + continue + data = "\n".join(data_lines) + if data.strip() == "[DONE]": + continue + try: + parsed = json.loads(data) + except ValueError: + continue + if isinstance(parsed, dict): + payloads.append(parsed) + return payloads + + def _jsonable(value: Any, seen: set[int]) -> Any: if hasattr(value, "model_dump"): try: diff --git a/tests/test_error_source_annotation.py b/tests/test_error_source_annotation.py new file mode 100644 index 00000000..cba04f8c --- /dev/null +++ b/tests/test_error_source_annotation.py @@ -0,0 +1,156 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SWITCH-882: failure-source annotation on error responses and the event log. + +Every client-facing error carries ``x-switchyard-error-source`` naming the +layer that originated it (``switchyard`` | ``provider``), plus +``x-switchyard-upstream-model`` when a routing selection had happened. The +backend-side ctx stamps are covered in ``test_latency_service_llm_backend.py``; +these tests cover the endpoint layer that renders them. +""" + +from __future__ import annotations + +import json +import logging + +import pytest + +from switchyard.lib.endpoints.dispatch import model_not_found_response +from switchyard.lib.endpoints.error_envelope import ( + ERROR_SOURCE_HEADER, + UPSTREAM_MODEL_HEADER, + error_response, + upstream_error_response, +) +from switchyard.lib.endpoints.upstream_error import ( + context_exhausted_response, + handle_chain_exception, + upstream_response_from_ctx, +) +from switchyard.lib.endpoints.upstream_error_log import log_upstream_attempt_failure +from switchyard.lib.proxy_context import ( + CTX_ERROR_SOURCE, + CTX_UPSTREAM_HTTP_BODY, + CTX_UPSTREAM_HTTP_STATUS, + CTX_UPSTREAM_MODEL, + ProxyContext, +) + +_UPSTREAM_401 = { + "error": {"message": "bad key", "type": "auth_error", "code": "invalid_api_key"} +} + + +# --- envelope builders ------------------------------------------------------- + + +def test_synthesized_envelope_defaults_to_switchyard_source() -> None: + resp = error_response(400, "bad", error_type="invalid_request_error", code="invalid_body") + assert resp.headers[ERROR_SOURCE_HEADER] == "switchyard" + assert UPSTREAM_MODEL_HEADER not in resp.headers + + +def test_upstream_envelope_labels_provider_and_upstream_model() -> None: + resp = upstream_error_response(429, _UPSTREAM_401, upstream_model="gpt-5") + assert resp.headers[ERROR_SOURCE_HEADER] == "provider" + assert resp.headers[UPSTREAM_MODEL_HEADER] == "gpt-5" + + +def test_model_not_found_labels_switchyard() -> None: + resp = model_not_found_response("nope") + assert resp.status_code == 404 + assert resp.headers[ERROR_SOURCE_HEADER] == "switchyard" + + +def test_context_exhausted_labels_switchyard() -> None: + resp = context_exhausted_response(RuntimeError("pool exhausted"), "openai") + assert resp.status_code == 400 + assert resp.headers[ERROR_SOURCE_HEADER] == "switchyard" + + +# --- ctx-driven recovery paths ---------------------------------------------- + + +def _ctx_with_upstream_stash(**extra: object) -> ProxyContext: + ctx = ProxyContext() + ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = 401 + ctx.metadata[CTX_UPSTREAM_HTTP_BODY] = _UPSTREAM_401 + for key, value in extra.items(): + ctx.metadata[key] = value + return ctx + + +def test_stashed_status_defaults_to_provider() -> None: + """A backend that stashes an upstream status without a source label gets + the passthrough default — the stash channel exists for provider errors.""" + resp = upstream_response_from_ctx(_ctx_with_upstream_stash()) + assert resp is not None + assert resp.status_code == 401 + assert resp.headers[ERROR_SOURCE_HEADER] == "provider" + assert UPSTREAM_MODEL_HEADER not in resp.headers + + +def test_stashed_switchyard_source_overrides_provider_default() -> None: + """caller_required-style rejections ride the upstream channel but must + surface as switchyard-originated.""" + ctx = _ctx_with_upstream_stash(**{CTX_ERROR_SOURCE: "switchyard"}) + resp = upstream_response_from_ctx(ctx) + assert resp is not None + assert resp.headers[ERROR_SOURCE_HEADER] == "switchyard" + + +def test_stashed_upstream_model_reaches_header() -> None: + ctx = _ctx_with_upstream_stash(**{CTX_UPSTREAM_MODEL: "gpt-5"}) + resp = upstream_response_from_ctx(ctx) + assert resp is not None + assert resp.headers[UPSTREAM_MODEL_HEADER] == "gpt-5" + + +def test_unexpected_internal_500_labels_switchyard() -> None: + resp = handle_chain_exception( + RuntimeError("boom"), ProxyContext(), inbound="openai", log_msg="test failure" + ) + assert resp.status_code == 500 + assert resp.headers[ERROR_SOURCE_HEADER] == "switchyard" + assert UPSTREAM_MODEL_HEADER not in resp.headers + + +def test_network_failure_500_labels_provider() -> None: + """A status-less upstream fault (network error after retries) renders as + the internal 500 envelope but is labeled provider via the ctx stamp.""" + ctx = ProxyContext() + ctx.metadata[CTX_ERROR_SOURCE] = "provider" + ctx.metadata[CTX_UPSTREAM_MODEL] = "gpt-5" + + resp = handle_chain_exception( + RuntimeError("connection reset"), ctx, inbound="openai", log_msg="test failure" + ) + + assert resp.status_code == 500 + assert json.loads(bytes(resp.body))["error"]["code"] == "internal_chain_error" + assert resp.headers[ERROR_SOURCE_HEADER] == "provider" + assert resp.headers[UPSTREAM_MODEL_HEADER] == "gpt-5" + + +# --- structured event log ---------------------------------------------------- + + +def test_attempt_failure_log_carries_upstream_model_and_source( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING, logger="switchyard.upstream_errors"): + log_upstream_attempt_failure( + model="route-id", + attempt=1, + status_code=429, + error=RuntimeError("rate limited"), + upstream_model="gpt-5", + ) + + record = json.loads(caplog.records[-1].message) + assert record["model"] == "route-id" + assert record["upstream_model"] == "gpt-5" + assert record["error_source"] == "provider" + assert record["code"] == "429" diff --git a/tests/test_latency_service_llm_backend.py b/tests/test_latency_service_llm_backend.py index 53657eb9..bae6d7e7 100644 --- a/tests/test_latency_service_llm_backend.py +++ b/tests/test_latency_service_llm_backend.py @@ -28,7 +28,12 @@ LatencyServiceBackendConfig, LatencyServiceEndpoint, ) -from switchyard.lib.proxy_context import ProxyContext +from switchyard.lib.proxy_context import ( + CTX_ERROR_SOURCE, + CTX_UPSTREAM_HTTP_STATUS, + CTX_UPSTREAM_MODEL, + ProxyContext, +) from switchyard.lib.switchyard import Switchyard from switchyard.server.switchyard_app import build_switchyard_app from switchyard_rust.core import ( @@ -783,6 +788,37 @@ async def test_responses_endpoint_dispatches_responses_natively(self): assert "messages" not in call_kwargs assert ctx.selected_model == "model-A" + async def test_responses_endpoint_preserves_exact_upstream_body(self): + """Same-format Responses passthrough returns the upstream JSON untouched. + + Provider-specific extras and explicit-null fields must survive the + wrap into ``ChatResponse`` (SWITCH-883) — nothing may be normalized, + re-synthesized, or dropped on this path. + """ + upstream_body = { + "id": "resp-raw", + "object": "response", + "created_at": 1719890000, + "model": "model-A", + "status": "completed", + "output": [], + "store": False, + "temperature": 1.0, + "top_p": 0.9, + "previous_response_id": None, + "provider_meta": {"region": "eastus"}, + } + backend = _make_backend(_config("model-A", request_type="openai_responses")) + backend._clients["model-A"].aresponses = AsyncMock(return_value=dict(upstream_body)) + + result = await backend.call( + ProxyContext(), + ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}), + ) + + assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_COMPLETION) + assert result.body == upstream_body + async def test_chat_endpoint_translates_responses_to_chat_fallback(self): backend = _make_backend(_config("model-A")) backend._clients["model-A"].acompletion = AsyncMock( @@ -1435,6 +1471,76 @@ async def test_metric_absent_until_traffic(self): assert "switchyard_latency_upstream_attempts_total" not in out +# --------------------------------------------------------------------------- +# Failure-source annotation (SWITCH-882) +# --------------------------------------------------------------------------- + + +class TestErrorSourceAnnotation: + """The backend stamps the failure origin + upstream model on ctx.""" + + async def test_http_failure_stamps_provider_source_and_upstream_model(self): + backend = _make_backend(_config("model-A")) + _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) + backend._clients["model-A"].acompletion = AsyncMock( + side_effect=_api_status_error(401) + ) + + ctx = ProxyContext() + with pytest.raises(openai.APIStatusError): + await backend.call(ctx, _openai_request()) + + assert ctx.metadata[CTX_ERROR_SOURCE] == "provider" + assert ctx.metadata[CTX_UPSTREAM_MODEL] == "model-A" + assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401 + + async def test_network_failure_stamps_provider_without_status(self): + """A status-less upstream fault is still provider-originated, so the + endpoint's 500 envelope can say so instead of implying an internal bug.""" + backend = _make_backend(_config("model-A")) + _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY}) + backend._clients["model-A"].acompletion = AsyncMock( + side_effect=RuntimeError("connection reset") + ) + + ctx = ProxyContext() + with pytest.raises(RuntimeError): + await backend.call(ctx, _openai_request()) + + assert ctx.metadata[CTX_ERROR_SOURCE] == "provider" + assert ctx.metadata[CTX_UPSTREAM_MODEL] == "model-A" + assert CTX_UPSTREAM_HTTP_STATUS not in ctx.metadata + + async def test_caller_required_rejection_is_switchyard_source(self): + """The caller_required 401 rides the upstream-status channel but is + Switchyard's own rejection — the source label must say so.""" + backend = _make_backend( + _config("model-A", credential_policy="caller_required") + ) + + ctx = ProxyContext() + with pytest.raises(PermissionError): + await backend.call(ctx, _openai_request()) + + assert ctx.metadata[CTX_ERROR_SOURCE] == "switchyard" + assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401 + assert CTX_UPSTREAM_MODEL not in ctx.metadata + + def test_translation_rejection_is_switchyard_source(self): + """The translation invalid-value 400 is likewise Switchyard-originated.""" + from switchyard.lib.backends.latency_service_llm_backend import ( + _stash_invalid_request_error, + ) + + ctx = ProxyContext() + _stash_invalid_request_error( + ctx, ValueError("InvalidValue: unsupported message role 'api'") + ) + + assert ctx.metadata[CTX_ERROR_SOURCE] == "switchyard" + assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 400 + + # --------------------------------------------------------------------------- # Readiness and shutdown # --------------------------------------------------------------------------- diff --git a/tests/test_latency_service_spans.py b/tests/test_latency_service_spans.py index 734798b6..fd36fe99 100644 --- a/tests/test_latency_service_spans.py +++ b/tests/test_latency_service_spans.py @@ -203,6 +203,9 @@ async def test_api_status_error_tags(tracer: _FakeTracer) -> None: assert attempt.tags["switchyard.upstream_status_code"] == 429 assert attempt.tags["switchyard.outcome"] == "retryable_error" assert attempt.tags["switchyard.error_code"] == "429" + # Failure-source annotation (SWITCH-882): the attempt failed upstream. + assert attempt.tags["switchyard.error_source"] == "provider" + assert attempt.tags["switchyard.upstream_model"] == "model-A" async def test_generic_error_tags(tracer: _FakeTracer) -> None: @@ -218,6 +221,9 @@ async def test_generic_error_tags(tracer: _FakeTracer) -> None: assert attempt.tags["switchyard.error_code"] == "none" # A non-HTTP failure has no status line. assert "switchyard.upstream_status_code" not in attempt.tags + # A network-level fault is still an upstream-side failure (SWITCH-882). + assert attempt.tags["switchyard.error_source"] == "provider" + assert attempt.tags["switchyard.upstream_model"] == "model-A" async def test_retry_count_is_sequential_and_ends_success(tracer: _FakeTracer) -> None: diff --git a/tests/test_live_stats_collector.py b/tests/test_live_stats_collector.py index 83eec6a1..ef7550ef 100644 --- a/tests/test_live_stats_collector.py +++ b/tests/test_live_stats_collector.py @@ -381,6 +381,39 @@ async def _events(): assert s.cache_read_tokens == 60 +async def test_openai_responses_tap_reads_usage_from_raw_sse_frames(): + """Verbatim-passthrough streams yield raw SSE frame strings; the tap must + still find the usage in the ``response.completed`` frame.""" + from switchyard.lib.chat_response.openai_responses import ResponsesApiStream + + collector = LiveStatsCollector() + proc = StatsResponseProcessor(collector) + + async def _frames(): + yield ( + 'event: response.output_text.delta\n' + 'data: {"type":"response.output_text.delta","delta":"hi"}\n\n' + ) + yield ": keep-alive\n\n" + yield ( + 'event: response.completed\n' + 'data: {"type":"response.completed","response":{"id":"resp-raw","usage":' + '{"input_tokens":300,"output_tokens":120,' + '"input_tokens_details":{"cached_tokens":60},' + '"output_tokens_details":{"reasoning_tokens":40}}}}\n\n' + ) + + response = ChatResponse.openai_responses_stream(ResponsesApiStream(_frames())) + await proc.process(_make_ctx(), response) + [_ async for _ in response.stream] + + s = collector.snapshot() + assert s.request_count == 1 + assert s.prompt_tokens == 300 + assert s.completion_tokens == 120 + assert s.cache_read_tokens == 60 + + # --------------------------------------------------------------------------- # Cost estimation — model name normalization # --------------------------------------------------------------------------- diff --git a/tests/test_llm_client.py b/tests/test_llm_client.py index 93166b91..b6d51ff9 100644 --- a/tests/test_llm_client.py +++ b/tests/test_llm_client.py @@ -5,9 +5,13 @@ from unittest.mock import AsyncMock, MagicMock +import httpx +import openai +import pytest +import respx from openai import AsyncOpenAI -from switchyard.lib.llm_client import OpenAILLMClient +from switchyard.lib.llm_client import OpenAILLMClient, RawSSEFrameStream def test_constructs_only_the_async_client() -> None: @@ -46,6 +50,12 @@ def _client_with_spied_options() -> tuple[OpenAILLMClient, MagicMock]: overridden.chat.completions.create = AsyncMock(return_value="overridden") overridden.responses.create = AsyncMock(return_value="responses-overridden") client.async_client.with_options.return_value = overridden + # Non-streaming ``aresponses`` fetches the raw HTTP response and + # returns its exact JSON body (SWITCH-883); wire raw spies per client. + for target, label in ((client.async_client, "base"), (overridden, "overridden")): + raw = MagicMock() + raw.http_response.json.return_value = {"src": f"responses-{label}"} + target.responses.with_raw_response.create = AsyncMock(return_value=raw) return client, client.async_client async def test_real_caller_key_uses_with_options(self) -> None: @@ -70,16 +80,126 @@ async def test_responses_real_caller_key_uses_with_options(self) -> None: client, async_client = self._client_with_spied_options() result = await client.aresponses(api_key="caller-key", model="m", input="hi") async_client.with_options.assert_called_once_with(api_key="caller-key") - assert result == "responses-overridden" + assert result == {"src": "responses-overridden"} async def test_responses_missing_key_falls_back_to_construction_key(self) -> None: client, async_client = self._client_with_spied_options() result = await client.aresponses(api_key=None, model="m", input="hi") async_client.with_options.assert_not_called() - assert result == "responses-base" + assert result == {"src": "responses-base"} async def test_responses_blank_key_falls_back_to_construction_key(self) -> None: client, async_client = self._client_with_spied_options() result = await client.aresponses(api_key=" ", model="m", input="hi") async_client.with_options.assert_not_called() - assert result == "responses-base" + assert result == {"src": "responses-base"} + + async def test_responses_streaming_uses_raw_streaming_path(self) -> None: + """Streaming fetches the raw SSE response (verbatim frames) and enters + the SDK context manager eagerly so error statuses raise at call time.""" + client, async_client = self._client_with_spied_options() + + async def _lines(): + yield "data: {}" + yield "" + + api_response = MagicMock() + api_response.http_response.aiter_lines = _lines + cm = MagicMock() + cm.__aenter__ = AsyncMock(return_value=api_response) + cm.__aexit__ = AsyncMock(return_value=False) + async_client.responses.with_streaming_response.create = MagicMock(return_value=cm) + + result = await client.aresponses(api_key=None, model="m", input="hi", stream=True) + + assert isinstance(result, RawSSEFrameStream) + cm.__aenter__.assert_awaited_once() + async_client.responses.create.assert_not_called() + async_client.responses.with_raw_response.create.assert_not_called() + await result.aclose() + cm.__aexit__.assert_awaited_once() + + +@respx.mock +async def test_aresponses_returns_exact_upstream_json() -> None: + """Non-streaming Responses calls return the upstream body as-is (SWITCH-883). + + Provider-specific extras and explicit-null fields must survive — the SDK + typed-model round-trip would normalize the former and ``exclude_none`` + serialization would drop the latter. + """ + upstream = { + "id": "resp_1", + "object": "response", + "created_at": 1719890000, + "model": "gpt-5", + "status": "completed", + "output": [], + "store": False, + "temperature": 1.0, + "top_p": 0.9, + "previous_response_id": None, + "provider_meta": {"azure_region": "eastus"}, + } + respx.post("http://upstream.test/v1/responses").mock( + return_value=httpx.Response(200, json=upstream) + ) + + client = OpenAILLMClient(api_key="k", base_url="http://upstream.test/v1") + result = await client.aresponses(model="gpt-5", input="hi") + + assert result == upstream + + +_SSE_FRAMES = [ + ( + 'event: response.created\n' + 'data: {"type":"response.created","response":{"id":"resp_1","store":false,' + '"temperature":1.0,"reasoning":{"effort":null},"provider_meta":{"az":"eastus"}}}\n\n' + ), + ": keep-alive\n\n", + ( + 'event: response.output_text.delta\n' + 'data: {"type":"response.output_text.delta","delta":"hi","vendor_extra":123}\n\n' + ), + ( + 'event: response.completed\n' + 'data: {"type":"response.completed","response":{"id":"resp_1","usage":' + '{"input_tokens":3,"output_tokens":2}}}\n\n' + ), +] + + +@respx.mock +async def test_aresponses_streaming_yields_verbatim_sse_frames() -> None: + """Streaming Responses calls yield the upstream SSE frames byte-for-byte + (SWITCH-883): unknown provider fields, explicit nulls, comment keep-alives, + and event names all survive because no typed-model parse happens. + """ + respx.post("http://upstream.test/v1/responses").mock( + return_value=httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content="".join(_SSE_FRAMES).encode(), + ) + ) + + client = OpenAILLMClient(api_key="k", base_url="http://upstream.test/v1") + stream = await client.aresponses(model="gpt-5", input="hi", stream=True) + + frames = [frame async for frame in stream] + assert frames == _SSE_FRAMES + + +@respx.mock +async def test_aresponses_streaming_error_status_raises_at_call_time() -> None: + """A non-2xx on a streaming Responses call raises ``APIStatusError`` from + ``aresponses`` itself — before any frame is consumed — so the backend's + retry/failover contract is unchanged by the raw-frame path.""" + respx.post("http://upstream.test/v1/responses").mock( + return_value=httpx.Response(500, json={"error": {"message": "boom"}}) + ) + + client = OpenAILLMClient(api_key="k", base_url="http://upstream.test/v1") + with pytest.raises(openai.APIStatusError): + await client.aresponses(model="gpt-5", input="hi", stream=True) diff --git a/tests/test_response_translation_engine.py b/tests/test_response_translation_engine.py index dd134e5e..7cc7cc06 100644 --- a/tests/test_response_translation_engine.py +++ b/tests/test_response_translation_engine.py @@ -351,3 +351,75 @@ async def test_reasoning_stream_events_validate_against_anthropic_sdk(self): assert signature_delta["delta"]["signature"] == "" RawContentBlockStartEvent.model_validate(thinking_start) RawContentBlockDeltaEvent.model_validate(signature_delta) + + +# --------------------------------------------------------------------------- +# Raw SSE frame parsing for cross-format stream translation (SWITCH-883) +# --------------------------------------------------------------------------- + + +class TestSseFramePayloads: + """``sse_frame_payloads`` feeds verbatim-passthrough frame strings back + into the translator; the SSE grammar corners must parse correctly.""" + + def test_single_frame_with_event_name(self): + from switchyard_rust.translation import sse_frame_payloads + + payloads = sse_frame_payloads( + 'event: response.created\ndata: {"type":"response.created"}\n\n' + ) + assert payloads == [{"type": "response.created"}] + + def test_multi_data_lines_join_with_newline(self): + from switchyard_rust.translation import sse_frame_payloads + + payloads = sse_frame_payloads('data: {"a":\ndata: 1}\n\n') + assert payloads == [{"a": 1}] + + def test_comment_done_and_non_json_frames_yield_nothing(self): + from switchyard_rust.translation import sse_frame_payloads + + assert sse_frame_payloads(": keep-alive\n\n") == [] + assert sse_frame_payloads("data: [DONE]\n\n") == [] + assert sse_frame_payloads("data: not-json\n\n") == [] + + def test_multiple_frames_in_one_string_parse_in_order(self): + from switchyard_rust.translation import sse_frame_payloads + + payloads = sse_frame_payloads('data: {"a":1}\n\ndata: {"b":2}\n\n') + assert payloads == [{"a": 1}, {"b": 2}] + + +async def test_translate_stream_accepts_raw_sse_frame_strings(): + """Cross-format translation parses raw Responses frames into chat chunks.""" + + async def _frames() -> AsyncIterator[str]: + yield ( + 'event: response.created\n' + 'data: {"type":"response.created","response":{"id":"r1","model":"m"}}\n\n' + ) + yield ( + 'event: response.output_text.delta\n' + 'data: {"type":"response.output_text.delta","delta":"hello"}\n\n' + ) + yield ": keep-alive\n\n" + yield ( + 'event: response.completed\n' + 'data: {"type":"response.completed","response":{"id":"r1","usage":' + '{"input_tokens":1,"output_tokens":1}}}\n\n' + ) + + chunks = [ + chunk + async for chunk in E.translate_stream( + "openai_responses", "openai_chat", _frames(), model="m", + ) + ] + + deltas = [ + choice.get("delta", {}).get("content") + for chunk in chunks + if isinstance(chunk, dict) + for choice in chunk.get("choices", []) + ] + assert "hello" in deltas diff --git a/tests/test_upstream_error_log.py b/tests/test_upstream_error_log.py index 7b3c3b94..7929fe75 100644 --- a/tests/test_upstream_error_log.py +++ b/tests/test_upstream_error_log.py @@ -51,15 +51,20 @@ def test_message_is_valid_json_with_expected_keys( "event", "timestamp", "model", + "upstream_model", "attempt", "status_code", "code", "outcome", + "error_source", "error_type", "error", } assert rec["event"] == EVENT_NAME == "upstream_attempt_failed" assert rec["model"] == "m" + # No upstream_model supplied → logged as null; source is constant. + assert rec["upstream_model"] is None + assert rec["error_source"] == "provider" assert rec["attempt"] == 1 assert rec["error_type"] == "RuntimeError" assert rec["error"] == "boom" diff --git a/tests/test_upstream_error_passthrough.py b/tests/test_upstream_error_passthrough.py index fd18e26b..1b717507 100644 --- a/tests/test_upstream_error_passthrough.py +++ b/tests/test_upstream_error_passthrough.py @@ -37,6 +37,10 @@ LatencyServiceBackendConfig, LatencyServiceEndpoint, ) +from switchyard.lib.endpoints.error_envelope import ( + ERROR_SOURCE_HEADER, + UPSTREAM_MODEL_HEADER, +) from switchyard.lib.endpoints.upstream_error import ( internal_chain_error_response, upstream_response_from_ctx, @@ -708,3 +712,336 @@ def test_anthropic_invalid_role_returns_400( f"body={response.text!r}" ) backend._clients["model-A"].acompletion.assert_not_awaited() + + +# --------------------------------------------------------------------------- +# Failure-source headers on the wire (SWITCH-882) +# --------------------------------------------------------------------------- +# The header tests above the endpoint layer call helpers directly; these prove +# the annotation actually survives the real FastAPI handlers + middleware to +# the HTTP response a client receives. + + +def test_upstream_401_header_labels_provider_on_the_wire( + latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend], +) -> None: + app, backend = latency_service_app + backend._clients["model-A"].acompletion = AsyncMock( + side_effect=_api_status_error(401, {"error": {"message": "bad key"}}), + ) + + with TestClient(app, raise_server_exceptions=False) as client: + response = client.post( + "/v1/chat/completions", + json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]}, + ) + + assert response.status_code == 401 + assert response.headers[ERROR_SOURCE_HEADER] == "provider" + assert response.headers[UPSTREAM_MODEL_HEADER] == "model-A" + # The provider body still passes through unmodified — annotation is header-only. + assert response.json()["error"]["message"] == "bad key" + + +def test_network_failure_500_header_labels_provider_on_the_wire( + latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend], +) -> None: + """A status-less upstream fault renders as the internal 500 envelope but + the header says provider — clients can tell it wasn't a Switchyard bug.""" + app, backend = latency_service_app + backend._clients["model-A"].acompletion = AsyncMock( + side_effect=RuntimeError("connection refused"), + ) + + with TestClient(app, raise_server_exceptions=False) as client: + response = client.post( + "/v1/chat/completions", + json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]}, + ) + + assert response.status_code == 500 + assert response.json()["error"]["code"] == "internal_chain_error" + assert response.headers[ERROR_SOURCE_HEADER] == "provider" + assert response.headers[UPSTREAM_MODEL_HEADER] == "model-A" + + +def test_invalid_role_400_header_labels_switchyard_on_the_wire( + latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend], +) -> None: + """The translation rejection rides the upstream-status channel but must + surface as switchyard-originated on the wire.""" + app, backend = latency_service_app + backend._clients["model-A"].acompletion = AsyncMock() + + with TestClient(app, raise_server_exceptions=False) as client: + response = client.post( + "/v1/responses", + json={ + "model": "model-A", + "input": [{"type": "message", "role": "api", "content": "ping"}], + }, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "invalid_value" + assert response.headers[ERROR_SOURCE_HEADER] == "switchyard" + + +def test_model_not_found_404_header_labels_switchyard_on_the_wire() -> None: + """RouteTable dispatch 404s carry the switchyard source header.""" + with _OpenAICompatStub() as upstream: + table = build_route_bundle_table({ + "defaults": { + "api_key": "k", + "base_url": upstream.base_url, + "format": "openai", + }, + "routes": { + "registered": { + "type": "model", + "target": "nvidia/nvidia/nemotron-nano-9b-v2", + } + }, + }) + + with TestClient(build_switchyard_app(table), raise_server_exceptions=False) as client: + response = client.post( + "/v1/chat/completions", + json={ + "model": "no-such-model", + "messages": [{"role": "user", "content": "hi"}], + }, + ) + + assert response.status_code == 404 + assert response.json()["error"]["code"] == "model_not_found" + assert response.headers[ERROR_SOURCE_HEADER] == "switchyard" + + +def test_caller_required_401_header_labels_switchyard_on_the_wire() -> None: + """The caller_required rejection is Switchyard's own 401: same status as a + provider auth failure, distinguishable by the source header on the wire.""" + config = LatencyServiceBackendConfig( + latency_service_url="http://latency-service.test:8080", + endpoints=[ + LatencyServiceEndpoint( + model="model-A", + api_key="svc-key", # pragma: allowlist secret + base_url="http://llm.test/v1", + ), + ], + credential_policy="caller_required", + ) + with patch( + "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", + ) as mock_cls: + mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") + with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"): + app = build_switchyard_app(_latency_service_switchyard(config)) + with TestClient(app, raise_server_exceptions=False) as client: + response = client.post( + "/v1/chat/completions", + json={ + "model": "model-A", + "messages": [{"role": "user", "content": "ping"}], + }, + ) + + assert response.status_code == 401 + assert response.json()["error"]["code"] == "missing_caller_api_key" + assert response.headers[ERROR_SOURCE_HEADER] == "switchyard" + assert UPSTREAM_MODEL_HEADER not in response.headers + + +# --------------------------------------------------------------------------- +# Responses API fidelity on the wire (SWITCH-883) +# --------------------------------------------------------------------------- + + +def test_responses_body_returned_exactly_on_the_wire() -> None: + """A Responses request through the full app returns the upstream JSON + byte-for-byte: provider extras and explicit-null fields included. + + This is the composed proof for SWITCH-883 — backend raw wrap, the + terminal translation short-circuit, and endpoint serialization together. + """ + upstream_body = { + "id": "resp-e2e", + "object": "response", + "created_at": 1719890000, + "model": "model-A", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg-1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "hello", "annotations": []}], + } + ], + "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}, + "store": False, + "temperature": 1.0, + "top_p": 0.9, + "text": {"format": {"type": "text"}}, + "reasoning": {"effort": None, "summary": None}, + "previous_response_id": None, + "provider_meta": {"azure_region": "eastus"}, + } + config = LatencyServiceBackendConfig( + latency_service_url="http://latency-service.test:8080", + endpoints=[ + LatencyServiceEndpoint( + model="model-A", + api_key="k", + base_url="http://llm.test/v1", + request_type="openai_responses", + ), + ], + ) + with patch( + "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", + ) as mock_cls: + mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") + with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"): + switchyard = _latency_service_switchyard(config) + backend = _find_latency_backend(switchyard) + backend._clients["model-A"].aresponses = AsyncMock( + return_value=dict(upstream_body) + ) + app = build_switchyard_app(switchyard) + with TestClient(app, raise_server_exceptions=False) as client: + response = client.post( + "/v1/responses", + json={"model": "model-A", "input": "ping"}, + ) + + assert response.status_code == 200, response.text + assert response.json() == upstream_body + + +def _latency_responses_app(frames: list[str]) -> FastAPI: + """Full app over one ``openai_responses`` endpoint streaming *frames*.""" + config = LatencyServiceBackendConfig( + latency_service_url="http://latency-service.test:8080", + endpoints=[ + LatencyServiceEndpoint( + model="model-A", + api_key="k", + base_url="http://llm.test/v1", + request_type="openai_responses", + ), + ], + ) + with patch( + "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient", + ) as mock_cls: + mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}") + with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"): + switchyard = _latency_service_switchyard(config) + backend = _find_latency_backend(switchyard) + + async def _frame_iter() -> AsyncIterator[str]: + for frame in frames: + yield frame + + backend._clients["model-A"].aresponses = AsyncMock( + return_value=_frame_iter() + ) + return build_switchyard_app(switchyard) + + +def test_responses_stream_forwarded_verbatim_on_the_wire() -> None: + """A same-format streaming Responses request returns the upstream SSE + frames byte-for-byte: unknown provider fields, explicit nulls, event + names, and comment keep-alives all survive (SWITCH-883 streaming leg).""" + frames = [ + ( + 'event: response.created\n' + 'data: {"type":"response.created","response":{"id":"resp_s1","store":false,' + '"temperature":1.0,"reasoning":{"effort":null,"summary":null}}}\n\n' + ), + ": keep-alive\n\n", + ( + 'event: response.output_text.delta\n' + 'data: {"type":"response.output_text.delta","delta":"hello","vendor_extra":1}\n\n' + ), + ( + 'event: response.completed\n' + 'data: {"type":"response.completed","response":{"id":"resp_s1","usage":' + '{"input_tokens":3,"output_tokens":2}}}\n\n' + ), + ] + app = _latency_responses_app(frames) + with TestClient(app, raise_server_exceptions=False) as client: + response = client.post( + "/v1/responses", + json={"model": "model-A", "input": "ping", "stream": True}, + ) + + assert response.status_code == 200, response.text + assert response.text == "".join(frames) + + +def test_azure_flavored_responses_stream_preserved_on_the_wire() -> None: + """Azure-shaped Responses events (content-filter annotations and other + provider extras) pass through the streaming path unmodified.""" + frames = [ + ( + 'event: response.output_text.delta\n' + 'data: {"type":"response.output_text.delta","delta":"hi",' + '"content_filter_results":{"hate":{"filtered":false,"severity":"safe"}}}\n\n' + ), + ( + 'event: response.completed\n' + 'data: {"type":"response.completed","response":{"id":"resp_az",' + '"prompt_filter_results":[{"prompt_index":0}],"usage":' + '{"input_tokens":1,"output_tokens":1},"previous_response_id":null}}\n\n' + ), + ] + app = _latency_responses_app(frames) + with TestClient(app, raise_server_exceptions=False) as client: + response = client.post( + "/v1/responses", + json={"model": "model-A", "input": "ping", "stream": True}, + ) + + assert response.status_code == 200, response.text + assert response.text == "".join(frames) + + +def test_responses_raw_frames_translate_for_chat_client_on_the_wire() -> None: + """Cross-format regression: a chat client served by a Responses endpoint + still gets translated chat chunks — raw frame strings are parsed back into + events for the translator instead of passing through.""" + frames = [ + ( + 'event: response.created\n' + 'data: {"type":"response.created","response":{"id":"resp_x","model":"m"}}\n\n' + ), + ( + 'event: response.output_text.delta\n' + 'data: {"type":"response.output_text.delta","delta":"hello"}\n\n' + ), + ( + 'event: response.completed\n' + 'data: {"type":"response.completed","response":{"id":"resp_x","usage":' + '{"input_tokens":3,"output_tokens":2}}}\n\n' + ), + ] + app = _latency_responses_app(frames) + with TestClient(app, raise_server_exceptions=False) as client: + response = client.post( + "/v1/chat/completions", + json={ + "model": "model-A", + "messages": [{"role": "user", "content": "ping"}], + "stream": True, + }, + ) + + assert response.status_code == 200, response.text + assert "chat.completion.chunk" in response.text + assert "hello" in response.text + assert "response.output_text.delta" not in response.text From ce38330b693762160142f6bbb623a47c6cf1e5c2 Mon Sep 17 00:00:00 2001 From: eric-liu-nvidia Date: Wed, 8 Jul 2026 02:31:08 +0800 Subject: [PATCH 05/11] feat(latency-service): pluggable shared session-affinity pin store with Redis L2 --- .agents/skills/switchyard-lib-core/SKILL.md | 1 + pyproject.toml | 11 +- switchyard/cli/route_bundle.py | 10 + switchyard/lib/affinity_pin_store.py | 39 +++ .../backends/latency_service_llm_backend.py | 77 ++++- .../config/latency_service_backend_config.py | 28 ++ .../llm_classifier/request_processor.py | 2 +- .../tier_selector_request_processor.py | 8 +- switchyard/lib/redis_pin_store.py | 101 ++++++ switchyard/lib/session_affinity.py | 176 ++++++++++- tests/e2e/test_latency_service_llm_backend.py | 18 +- tests/test_latency_service_health_metrics.py | 10 +- tests/test_latency_service_llm_backend.py | 79 ++++- .../test_llm_classifier_request_processor.py | 10 +- tests/test_redis_pin_store.py | 121 +++++++ tests/test_route_bundle.py | 31 ++ tests/test_session_affinity.py | 294 ++++++++++++++++++ uv.lock | 33 +- 18 files changed, 988 insertions(+), 61 deletions(-) create mode 100644 switchyard/lib/affinity_pin_store.py create mode 100644 switchyard/lib/redis_pin_store.py create mode 100644 tests/test_redis_pin_store.py create mode 100644 tests/test_session_affinity.py diff --git a/.agents/skills/switchyard-lib-core/SKILL.md b/.agents/skills/switchyard-lib-core/SKILL.md index 75e13e4e..de8b3b05 100644 --- a/.agents/skills/switchyard-lib-core/SKILL.md +++ b/.agents/skills/switchyard-lib-core/SKILL.md @@ -39,6 +39,7 @@ right validation set. If the change is driven by a launcher need, also read | An OpenAI-compatible provider target such as NVIDIA Inference Hub or OpenRouter | Use the existing OpenAI-compatible backend/profile with `base_url`, `api_key`, and model id wiring. Add a new backend only when the provider has a real wire-format, auth, retry, or health contract that cannot fit that path. | | Direct Rust component bindings | Add concrete PyO3 classes under `crates/switchyard-py/src/component_bindings/`, keep config bindings near the component binding that consumes them, and expose them lazily from `switchyard_rust/components.py`. Do not keep growing `core_bindings.rs` or `switchyard_rust/core.py` with concrete component classes. | | Route YAML / model dispatch | Use `switchyard/cli/route_bundle.py` and `switchyard/lib/route_table_builders.py`. They build `RouteTable` entries from profile-backed runtimes and keep launchers plus `switchyard serve --routing-profiles` on one path. | +| Shared/persistent session-affinity pins across workers or pod churn | Configure the latency route with `session_affinity: true` + `affinity_store: redis` + `affinity_store_url` (optional `affinity_store_ttl_seconds`, `affinity_key_prefix`). `SessionAffinity` keeps the Rust `SessionCache` as L1 and reads/writes through the `AffinityPinStore` L2 (`switchyard/lib/redis_pin_store.py`), fail-open behind a 0.1s socket timeout and a 3-failure/10s-cooldown circuit breaker (`switchyard_affinity_l2_breaker_open` gauge). Requires the `switchyard[affinity-redis]` extra. | | Stats / telemetry | Reuse `StatsRequestProcessor`, `StatsResponseProcessor`, `StatsLlmBackend`, and `StatsAccumulator`. A profile config should thread one accumulator through all three when stats are enabled. Do not write a parallel collector. | | A fixed-path endpoint contributed by per-route components | Set `Endpoint.register_once = True`; `build_switchyard_app(...)` mounts the first instance while still running every component's lifecycle. Leave the default `False` for configurable endpoint classes that may mount distinct instances. | | Per-endpoint attribution on `/metrics` for a Python backend that can't be wrapped by `StatsLlmBackend` | Set `ctx.selected_model = endpoint_id` before returning the response. Also set `ctx.backend_call_latency_ms = upstream_call_ms` so the response processor can compute routing overhead. `LatencyServiceLLMBackend.call` is the reference. | diff --git a/pyproject.toml b/pyproject.toml index f3945e60..14e3aee6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,9 +82,18 @@ intake = [ "nemo-platform>=0.1.0,<1.0", ] +# Shared session-affinity pin store. redis-py ships ``redis.asyncio`` natively, +# so no separate async client is needed. Lazy-imported by +# ``switchyard.lib.redis_pin_store`` — the default install never pulls it in. +# Floor 5.0.1: first release with ``aclose()`` on async clients, which the +# store's shutdown path calls. +affinity-redis = [ + "redis>=5.0.1,<6", +] + # Everything — all optional dependencies for full-featured deployment. all = [ - "nemo-switchyard[server,cli,gpu,tracing,intake]", + "nemo-switchyard[server,cli,gpu,tracing,intake,affinity-redis]", ] # Dev tooling lives in a PEP 735 dependency group rather than an optional diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py index eadd6a92..c41a13a8 100644 --- a/switchyard/cli/route_bundle.py +++ b/switchyard/cli/route_bundle.py @@ -215,6 +215,10 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: "enable_stats", "session_affinity", "affinity_max_sessions", + "affinity_store", + "affinity_store_url", + "affinity_store_ttl_seconds", + "affinity_key_prefix", }) | _LATENCY_ENDPOINT_DEFAULT_KEYS _NOOP_ROUTE_KEYS = _ROUTE_METADATA_KEYS _DETERMINISTIC_ROUTE_KEYS = ( @@ -1168,6 +1172,12 @@ def _latency_service_switchyard( "affinity_max_sessions": _optional_int( route.get("affinity_max_sessions"), default=10_000 ), + "affinity_store": _optional_str(route.get("affinity_store")) or "memory", + "affinity_store_url": _optional_str(route.get("affinity_store_url")), + "affinity_store_ttl_seconds": _optional_int( + route.get("affinity_store_ttl_seconds"), default=3_600 + ), + "affinity_key_prefix": _optional_str(route.get("affinity_key_prefix")) or "swyd:pin:", }) return ProfileSwitchyard( LatencyServiceProfileConfig.from_config(config) diff --git a/switchyard/lib/affinity_pin_store.py b/switchyard/lib/affinity_pin_store.py new file mode 100644 index 00000000..1654bfeb --- /dev/null +++ b/switchyard/lib/affinity_pin_store.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pluggable async L2 store for session-affinity pins. + +The in-process Rust ``SessionCache`` stays the L1 inside +:class:`switchyard.lib.session_affinity.SessionAffinity`. An optional object +implementing this protocol is the L2: a shared, out-of-process store (e.g. +Redis) that lets pins be read by every Switchyard worker/pod and survive pod +churn, so a conversation keeps hitting the same upstream even when its turns +land on different replicas. +""" + +from __future__ import annotations + +from typing import Protocol, runtime_checkable + + +@runtime_checkable +class AffinityPinStore(Protocol): + """Shared, out-of-process store for conversation→decision pins. + + The pinned value is opaque to the store (an endpoint id for the latency + backend, a tier label for the classifier router). Implementations are + called on the request path, so they should be fast, and + :class:`SessionAffinity` treats them as **best-effort**: an error or timeout + must never break request routing (the caller falls back to L1 / unpinned). + """ + + async def get(self, key: str) -> str | None: + """Return the value pinned to ``key``, or ``None`` if absent.""" + ... + + async def put(self, key: str, value: str) -> None: + """Pin ``key`` to ``value``. May apply a TTL or eviction policy.""" + ... + + +__all__ = ["AffinityPinStore"] diff --git a/switchyard/lib/backends/latency_service_llm_backend.py b/switchyard/lib/backends/latency_service_llm_backend.py index 81983aac..7f5f96eb 100644 --- a/switchyard/lib/backends/latency_service_llm_backend.py +++ b/switchyard/lib/backends/latency_service_llm_backend.py @@ -28,6 +28,7 @@ from openai import APIStatusError, AsyncStream +from switchyard.lib.affinity_pin_store import AffinityPinStore from switchyard.lib.backends.health_poller import ( EndpointHealth, EndpointHealthStatus, @@ -131,10 +132,13 @@ def __init__( # Session affinity: pins a conversation to the endpoint that last served # it so its upstream prompt/KV cache stays warm. Shared coordinator with # the classifier router; the latency-specific reuse policy (health-gated, - # failover-aware) lives in ``_select_endpoint_decision``. + # failover-aware) lives in ``_select_endpoint_decision``. An optional + # shared L2 (Redis) lets pins outlive this worker and be read by peers, + # so a conversation stays pinned across replicas and pod churn. self._affinity = SessionAffinity( enabled=config.session_affinity, max_sessions=config.affinity_max_sessions, + l2=_build_affinity_l2(config), ) # Cumulative warm-reuse counters, published on /metrics when affinity is # enabled. A "hit" is a turn served by an existing pin; a "miss" is a @@ -363,7 +367,7 @@ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: # Resolve the session-affinity pin once (keyed on the stable conversation # prefix). ``None`` when affinity is disabled or the conversation isn't # pinned yet, so every attempt routes purely by health + latency. - pinned_endpoint = self._affinity.pinned(ctx, request) + pinned_endpoint = await self._affinity.pinned(ctx, request) last_exc: Exception | None = None # Upstream model of the last failed attempt, surfaced on the error @@ -541,7 +545,7 @@ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: # follows a recovery: if the previous pin degraded and we # re-routed, the endpoint that worked becomes the new pin. # (No-op when affinity is disabled.) - self._affinity.pin(ctx, request, model_id) + await self._affinity.pin(ctx, request, model_id) return _chat_response_for_request_type(target_request_type, result) @@ -653,15 +657,17 @@ def _record_model_attempt( # -- Lifecycle ---------------------------------------------------------- - def shutdown(self) -> None: - """Stop the background :class:`HealthPoller` daemon. + async def shutdown(self) -> None: + """Stop the background :class:`HealthPoller` daemon and release the + shared session-affinity store, when one is configured. - Picked up automatically by ``NemoSwitchyardServer``'s component - teardown hook (see ``server.py``'s lifespan context manager). - Safe to call multiple times. + Picked up automatically by the server's component teardown hook + (``_run_lifecycle_method``), which awaits awaitable ``shutdown`` + results. Safe to call multiple times. """ prometheus_emitter.unregister(self._render_prometheus_lines) self._poller.stop() + await self._affinity.aclose() def is_ready(self) -> bool: """True once the background poller has completed at least one successful poll.""" @@ -773,6 +779,42 @@ def _render_prometheus_lines(self) -> list[str]: lines.append("# TYPE switchyard_affinity_misses_total counter") lines.append(f"switchyard_affinity_misses_total {self._affinity_misses}") + # Shared-store (L2) counters — emitted only when a shared pin store + # is configured. Hits measure cross-worker/churn pin reuse; errors + # count fail-open store operations (the alerting signal for a store + # that is degraded while requests keep succeeding). + if self._affinity.l2_enabled: + lines.append( + "# HELP switchyard_affinity_l2_hits_total " + "Pins resolved from the shared (L2) affinity store after an " + "in-process (L1) miss — cross-worker warm reuse." + ) + lines.append("# TYPE switchyard_affinity_l2_hits_total counter") + lines.append( + f"switchyard_affinity_l2_hits_total {self._affinity.l2_hits}" + ) + + lines.append( + "# HELP switchyard_affinity_l2_errors_total " + "Shared (L2) affinity-store operations that failed open " + "(get or put); routing fell back to in-process pins only." + ) + lines.append("# TYPE switchyard_affinity_l2_errors_total counter") + lines.append( + f"switchyard_affinity_l2_errors_total {self._affinity.l2_errors}" + ) + + lines.append( + "# HELP switchyard_affinity_l2_breaker_open " + "1 while the shared-store circuit breaker is open " + "(operations skipped without a network attempt); 0 when closed." + ) + lines.append("# TYPE switchyard_affinity_l2_breaker_open gauge") + lines.append( + "switchyard_affinity_l2_breaker_open " + f"{int(self._affinity.l2_breaker_open)}" + ) + # Per-model upstream-attempt breakdown — the latency-service-scoped # complement to the layer-aggregate ``switchyard_upstream_attempts_total``. # Series are created lazily per (requested_model, upstream_model, outcome, @@ -853,6 +895,25 @@ def _affinity_usable(health: EndpointHealth | None) -> bool: ) +def _build_affinity_l2(config: LatencyServiceBackendConfig) -> AffinityPinStore | None: + """Build the optional shared L2 pin store from config (``None`` = L1-only). + + ``affinity_store="memory"`` (the default) keeps pins per process. ``"redis"`` + imports :class:`RedisPinStore` lazily so the ``redis`` dependency stays + optional. Config validation guarantees a URL when the store is Redis. + """ + if config.affinity_store != "redis": + return None + from switchyard.lib.redis_pin_store import RedisPinStore + + assert config.affinity_store_url is not None # enforced by config validator + return RedisPinStore( + config.affinity_store_url, + ttl_seconds=config.affinity_store_ttl_seconds, + key_prefix=config.affinity_key_prefix, + ) + + # Prefix of the stringified translation error for a client-invalid payload. # ``TranslationError::kind()`` is the stable FFI signal for the error variant # (``crates/switchyard-translation/src/error.rs``); ``py_translation_error`` diff --git a/switchyard/lib/config/latency_service_backend_config.py b/switchyard/lib/config/latency_service_backend_config.py index dd27b883..f2d93ed5 100644 --- a/switchyard/lib/config/latency_service_backend_config.py +++ b/switchyard/lib/config/latency_service_backend_config.py @@ -108,6 +108,16 @@ class LatencyServiceBackendConfig(BaseModel): its endpoint degrades or the call fails. Per process. Default off. affinity_max_sessions: Bounded-LRU cap on pinned conversations; ignored when ``session_affinity`` is off. + affinity_store: Shared L2 pin store behind the in-process LRU. ``"memory"`` + (default) keeps pins per process; ``"redis"`` shares them across + workers/pods and persists them across pod churn. The store is + best-effort — an L2 error never fails a request. + affinity_store_url: Connection URL for the shared store (e.g. + ``"redis://host:6379/0"``). Required when ``affinity_store`` is + ``"redis"``. + affinity_store_ttl_seconds: Expiry for a shared pin. The backend re-pins + on every successful turn, so an active conversation slides its TTL. + affinity_key_prefix: Namespace prefix for shared-store keys. enable_stats: When ``True`` (default), the factory wires a :class:`StatsRequestProcessor` + :class:`StatsResponseProcessor` pair sharing one :class:`StatsAccumulator` and wraps the @@ -128,6 +138,10 @@ class LatencyServiceBackendConfig(BaseModel): credential_policy: LatencyServiceCredentialPolicy = "configured_endpoint" session_affinity: bool = False affinity_max_sessions: int = Field(default=10_000, ge=0) + affinity_store: Literal["memory", "redis"] = "memory" + affinity_store_url: str | None = None + affinity_store_ttl_seconds: int = Field(default=3_600, gt=0) + affinity_key_prefix: str = "swyd:pin:" enable_stats: bool = True @model_validator(mode="after") @@ -138,3 +152,17 @@ def _affinity_capacity_nonzero_when_enabled(self) -> Self: "affinity_max_sessions must be > 0 when session_affinity is enabled" ) return self + + @model_validator(mode="after") + def _redis_store_requires_url_and_affinity(self) -> Self: + # A shared store is dead config unless affinity is on and reachable. + if self.affinity_store == "redis": + if not self.session_affinity: + raise ValueError( + 'affinity_store="redis" requires session_affinity to be enabled' + ) + if not self.affinity_store_url: + raise ValueError( + 'affinity_store="redis" requires affinity_store_url to be set' + ) + return self diff --git a/switchyard/lib/processors/llm_classifier/request_processor.py b/switchyard/lib/processors/llm_classifier/request_processor.py index cae387b0..6196fded 100644 --- a/switchyard/lib/processors/llm_classifier/request_processor.py +++ b/switchyard/lib/processors/llm_classifier/request_processor.py @@ -341,7 +341,7 @@ def __init__( ) async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: - if self._affinity is not None and self._affinity.pinned(ctx, request) is not None: + if self._affinity is not None and (await self._affinity.pinned(ctx, request)) is not None: # Already pinned: the selector reuses the pin and ignores any # verdict, so skip the LLM call — classify once per task, not per turn. return request diff --git a/switchyard/lib/processors/llm_classifier/tier_selector_request_processor.py b/switchyard/lib/processors/llm_classifier/tier_selector_request_processor.py index 55c4648b..f05ea8c1 100644 --- a/switchyard/lib/processors/llm_classifier/tier_selector_request_processor.py +++ b/switchyard/lib/processors/llm_classifier/tier_selector_request_processor.py @@ -201,7 +201,7 @@ def __init__( async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: signals = _read_signals(ctx) - decision = self._decide(ctx, request, signals) + decision = await self._decide(ctx, request, signals) ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = decision.tier ctx.metadata[CTX_DETERMINISTIC_TIER_DECISION] = decision @@ -217,14 +217,14 @@ async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: ) return request - def _decide( + async def _decide( self, ctx: ProxyContext, request: ChatRequest, signals: RouteDecision | None ) -> TierSelectionDecision: """Select a tier, reusing a per-conversation pin when affinity is on. On a reused pin the classifier was skipped upstream, so ``signals`` may be absent — the pin alone decides the tier.""" - pinned = self._affinity.pinned(ctx, request) if self._affinity else None + pinned = await self._affinity.pinned(ctx, request) if self._affinity else None if pinned is not None: return TierSelectionDecision( tier=pinned, @@ -235,7 +235,7 @@ def _decide( # Pin only a confident verdict — pinning a fail-open fallback would lock # the task to the default tier and the classifier would never re-run. if self._affinity is not None and decision.source in _STICKY_SOURCES: - self._affinity.pin(ctx, request, decision.tier) + await self._affinity.pin(ctx, request, decision.tier) return decision def _select(self, signals: RouteDecision | None) -> TierSelectionDecision: diff --git a/switchyard/lib/redis_pin_store.py b/switchyard/lib/redis_pin_store.py new file mode 100644 index 00000000..66b76e6a --- /dev/null +++ b/switchyard/lib/redis_pin_store.py @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Redis-backed shared L2 store for session-affinity pins. + +Implements :class:`~switchyard.lib.affinity_pin_store.AffinityPinStore` against a +standalone Redis so pins are visible to every Switchyard worker/pod and survive +pod churn. Keys are namespaced ``{key_prefix}{session_key}`` and expire after +``ttl_seconds``; the latency backend re-pins on every successful turn, so an +active conversation slides its own TTL. + +The ``redis`` dependency is optional (``switchyard[affinity-redis]``) and is +imported lazily, so the default install never pulls it in. Socket timeouts are +bounded: a slow or unreachable store fails fast into ``SessionAffinity``'s +best-effort (fail-open) path rather than blocking the request. +""" + +from __future__ import annotations + +from typing import Any + +#: Bounded socket timeout (seconds). The store is expected to be colocated with +#: the workers, so a healthy round-trip is sub-millisecond — 100 ms is ~100× +#: headroom. The cap bounds how long a *degraded* store can stall a request +#: before fail-open kicks in (and before SessionAffinity's circuit breaker +#: stops attempting the store at all). +_DEFAULT_SOCKET_TIMEOUT_S = 0.1 + + +class RedisPinStore: + """Shared, persistent pin store backed by a standalone Redis. + + Args: + url: Redis connection URL (e.g. ``"redis://host:6379/0"``). + ttl_seconds: Expiry applied to each pin write. Must be > 0. + key_prefix: Namespace prepended to every key. + socket_timeout: Per-operation and connect timeout, in seconds. + client: Pre-built async Redis client. Primarily for tests; when ``None`` + the client is created lazily from ``url`` on first use. + """ + + def __init__( + self, + url: str, + *, + ttl_seconds: int = 3_600, + key_prefix: str = "swyd:pin:", + socket_timeout: float = _DEFAULT_SOCKET_TIMEOUT_S, + client: Any = None, + ) -> None: + if ttl_seconds <= 0: + raise ValueError("ttl_seconds must be > 0") + self._url = url + self._ttl_seconds = ttl_seconds + self._key_prefix = key_prefix + self._socket_timeout = socket_timeout + self._client = client + + def _get_client(self) -> Any: + """Return the async Redis client, building it lazily on first use.""" + if self._client is None: + # Lazy import keeps ``redis`` an optional dependency. + from redis.asyncio import from_url + + # redis-py's from_url is untyped; keep strict mypy green for devs + # who install the optional extra. + self._client = from_url( # type: ignore[no-untyped-call] + self._url, + decode_responses=True, + socket_timeout=self._socket_timeout, + socket_connect_timeout=self._socket_timeout, + ) + return self._client + + def _redis_key(self, key: str) -> str: + return f"{self._key_prefix}{key}" + + async def get(self, key: str) -> str | None: + """Return the pinned value for ``key``, or ``None`` if absent/expired.""" + value = await self._get_client().get(self._redis_key(key)) + return value if value is None else str(value) + + async def put(self, key: str, value: str) -> None: + """Pin ``key`` to ``value`` with the configured TTL.""" + await self._get_client().set(self._redis_key(key), value, ex=self._ttl_seconds) + + async def aclose(self) -> None: + """Close the client and its connection pool. Safe to call repeatedly. + + Releases the client reference first, so a second call is a no-op (a + later ``get``/``put`` would lazily rebuild — but the store is only + closed at backend shutdown). + """ + client = self._client + if client is None: + return + self._client = None + await client.aclose() + + +__all__ = ["RedisPinStore"] diff --git a/switchyard/lib/session_affinity.py b/switchyard/lib/session_affinity.py index c0598481..eca46d2e 100644 --- a/switchyard/lib/session_affinity.py +++ b/switchyard/lib/session_affinity.py @@ -5,12 +5,30 @@ from __future__ import annotations +import inspect +import logging +import time +from typing import TYPE_CHECKING + from switchyard.lib.conversation_turn import conversation_turn_number from switchyard.lib.proxy_context import ProxyContext from switchyard.lib.session_cache import SessionCache from switchyard.lib.session_key import session_key_from_body from switchyard_rust.core import ChatRequest +if TYPE_CHECKING: + from collections.abc import Callable + + from switchyard.lib.affinity_pin_store import AffinityPinStore + +logger = logging.getLogger(__name__) + +#: Consecutive L2 failures that open the circuit breaker. +_L2_BREAKER_THRESHOLD = 3 + +#: Seconds the breaker stays open before allowing a recovery probe. +_L2_BREAKER_COOLDOWN_S = 10.0 + #: ``ProxyContext.metadata`` key memoizing the per-request session key, so #: callers that consult affinity more than once don't re-hash the (growing) #: request body. @@ -27,6 +45,17 @@ class SessionAffinity: classifier router, an endpoint id for the latency backend. Each caller keeps its own *policy* (when to write a pin, whether to honor one) on top. + The in-process ``SessionCache`` is the L1. An optional ``l2`` implementing + :class:`~switchyard.lib.affinity_pin_store.AffinityPinStore` is a shared, + out-of-process store (e.g. Redis): pins are read through it on an L1 miss + (and warmed back into L1) and written through it, so a conversation stays + pinned across workers and pod churn. The L2 is **best-effort** — any error + or timeout falls back to L1 / unpinned and never breaks routing — and sits + behind a circuit breaker: after ``l2_breaker_threshold`` consecutive + failures, L2 operations are skipped (no network attempt, zero added + latency) for ``l2_breaker_cooldown_s``; the first operation after the + cooldown is the recovery probe, whose success closes the breaker. + A disabled instance no-ops (and never hashes the body). Not thread-safe — touched only on the request path. """ @@ -37,12 +66,38 @@ def __init__( enabled: bool = False, max_sessions: int = 10_000, warmup_turns: int = 0, + l2: AffinityPinStore | None = None, + l2_breaker_threshold: int = _L2_BREAKER_THRESHOLD, + l2_breaker_cooldown_s: float = _L2_BREAKER_COOLDOWN_S, + clock: Callable[[], float] = time.monotonic, ) -> None: if warmup_turns < 0: raise ValueError("warmup_turns must be >= 0") + if l2_breaker_threshold < 1: + raise ValueError("l2_breaker_threshold must be >= 1") + if l2_breaker_cooldown_s <= 0: + raise ValueError("l2_breaker_cooldown_s must be > 0") self._enabled = enabled self._warmup_turns = warmup_turns self._pins: SessionCache = SessionCache(max_sessions) + #: Optional shared/persistent L2 store; ``None`` keeps behavior L1-only. + self._l2 = l2 + # L2 observability: hits = pins rescued from the shared store after an + # L1 miss (cross-worker reuse); errors = fail-open get/put operations + # (the alerting signal for a silently degraded store). Event-loop only, + # like the pins themselves — no lock. + self._l2_hits = 0 + self._l2_errors = 0 + # Circuit breaker over the L2: a streak of consecutive failures opens + # it, and while open every operation is skipped without a network + # attempt, so a store outage stops taxing requests with timeout waits. + # ``clock`` is injectable for tests; monotonic so wall-clock jumps + # can't wedge the breaker. + self._l2_breaker_threshold = l2_breaker_threshold + self._l2_breaker_cooldown_s = l2_breaker_cooldown_s + self._clock = clock + self._l2_failure_streak = 0 + self._l2_skip_until = 0.0 @property def enabled(self) -> bool: @@ -58,17 +113,124 @@ def warmup_turns(self) -> int: """Number of initial turns that cannot read or write affinity pins.""" return self._warmup_turns - def pinned(self, ctx: ProxyContext, request: ChatRequest) -> str | None: - """Return the value pinned to ``request``'s conversation, or ``None``.""" + @property + def l2_enabled(self) -> bool: + """Whether a shared (L2) pin store is configured.""" + return self._l2 is not None + + @property + def l2_hits(self) -> int: + """Pins resolved from the shared store after an in-process (L1) miss.""" + return self._l2_hits + + @property + def l2_errors(self) -> int: + """Shared-store operations (get or put) that failed open.""" + return self._l2_errors + + @property + def l2_breaker_open(self) -> bool: + """Whether the breaker is currently skipping shared-store operations.""" + return self._l2 is not None and self._clock() < self._l2_skip_until + + def _l2_skips(self) -> bool: + """Breaker gate checked before every L2 operation. + + ``True`` while the cooldown is running — skip the store without a + network attempt. Once the cooldown elapses this returns ``False``, so + the next operation goes through as the recovery probe (concurrent + in-flight requests at that instant may each probe; every probe is + bounded by the store's socket timeout, so the burst is harmless). + """ + return self._clock() < self._l2_skip_until + + def _record_l2_failure(self, what: str) -> None: + """Count one fail-open L2 operation and open the breaker at the streak.""" + self._l2_errors += 1 + self._l2_failure_streak += 1 + if self._l2_failure_streak < self._l2_breaker_threshold: + logger.warning("session-affinity L2 %s", what, exc_info=True) + return + self._l2_skip_until = self._clock() + self._l2_breaker_cooldown_s + logger.warning( + "session-affinity L2 %s; breaker open for %.1fs (failure streak %d)", + what, + self._l2_breaker_cooldown_s, + self._l2_failure_streak, + exc_info=True, + ) + + def _record_l2_success(self) -> None: + """Reset the failure streak; close the breaker after a successful probe.""" + if self._l2_failure_streak >= self._l2_breaker_threshold: + logger.info("session-affinity L2 recovered; breaker closed") + self._l2_failure_streak = 0 + self._l2_skip_until = 0.0 + + async def pinned(self, ctx: ProxyContext, request: ChatRequest) -> str | None: + """Return the value pinned to ``request``'s conversation, or ``None``. + + Checks L1 first; on a miss (and only when an L2 is configured) consults + the shared store and warms the hit back into L1 so later turns on this + worker skip the round-trip. An L2 error is swallowed — the conversation + is treated as unpinned rather than failing the request — and while the + breaker is open the store isn't consulted at all. + """ if not self._enabled or self._is_warmup_turn(request): return None - return self._pins.get(self._session_key(ctx, request)) - - def pin(self, ctx: ProxyContext, request: ChatRequest, value: str) -> None: - """Pin ``request``'s conversation to ``value`` (no-op when disabled).""" + key = self._session_key(ctx, request) + value = self._pins.get(key) + if value is not None or self._l2 is None or self._l2_skips(): + return value + try: + value = await self._l2.get(key) + except Exception: + self._record_l2_failure("get failed; routing without a pin") + return None + self._record_l2_success() + if value is not None: + self._l2_hits += 1 + self._pins.put(key, value) + return value + + async def pin(self, ctx: ProxyContext, request: ChatRequest, value: str) -> None: + """Pin ``request``'s conversation to ``value`` (no-op when disabled). + + Writes through to the L2 when configured; an L2 error is swallowed so + the pin stays worker-local rather than failing the request, and while + the breaker is open the write is skipped entirely. + """ if not self._enabled or self._is_warmup_turn(request): return - self._pins.put(self._session_key(ctx, request), value) + key = self._session_key(ctx, request) + self._pins.put(key, value) + if self._l2 is None or self._l2_skips(): + return + try: + await self._l2.put(key, value) + except Exception: + self._record_l2_failure("put failed; pin is worker-local only") + return + self._record_l2_success() + + async def aclose(self) -> None: + """Release the shared store's resources (no-op for L1-only instances). + + Duck-typed: closes the L2 only if it exposes ``aclose()`` (the protocol + doesn't require one). Best-effort like every other L2 interaction — a + close failure is logged, never raised, so backend teardown can't wedge. + """ + if self._l2 is None: + return + closer = getattr(self._l2, "aclose", None) + if not callable(closer): + return + try: + result = closer() + if inspect.isawaitable(result): + await result + except Exception: + logger.warning("session-affinity L2 close failed", exc_info=True) def __len__(self) -> int: """Number of conversations currently pinned.""" diff --git a/tests/e2e/test_latency_service_llm_backend.py b/tests/e2e/test_latency_service_llm_backend.py index 0bc7ee70..fbd93dc5 100644 --- a/tests/e2e/test_latency_service_llm_backend.py +++ b/tests/e2e/test_latency_service_llm_backend.py @@ -233,7 +233,7 @@ async def test_healthy_endpoint_serves_request(self, mock_latency_service): assert response.body["choices"][0]["message"]["role"] == "assistant" assert ctx.selected_model == MODEL_A finally: - backend.shutdown() + await backend.shutdown() async def test_response_has_usage(self, mock_latency_service): """Response should include token usage statistics.""" @@ -250,7 +250,7 @@ async def test_response_has_usage(self, mock_latency_service): assert response.body["usage"]["prompt_tokens"] > 0 assert response.body["usage"]["completion_tokens"] > 0 finally: - backend.shutdown() + await backend.shutdown() # ================================================================== # @@ -284,7 +284,7 @@ async def test_streaming_yields_chunks(self, mock_latency_service): full_text = "".join(text_parts) assert len(full_text) > 0 finally: - backend.shutdown() + await backend.shutdown() # ================================================================== # @@ -315,7 +315,7 @@ async def test_healthy_preferred_over_degraded(self, mock_latency_service): f"Expected all traffic to HEALTHY model, got: {selected_models}" ) finally: - backend.shutdown() + await backend.shutdown() async def test_unknown_falls_back_to_random(self, mock_latency_service): """When all endpoints are UNKNOWN, the backend should distribute @@ -336,7 +336,7 @@ async def test_unknown_falls_back_to_random(self, mock_latency_service): f"Expected random distribution across models, got: {selected_models}" ) finally: - backend.shutdown() + await backend.shutdown() async def test_poller_receives_health_updates(self, mock_latency_service): """The background poller should pick up health changes from the @@ -354,7 +354,7 @@ async def test_poller_receives_health_updates(self, mock_latency_service): assert backend._health_cache[MODEL_A].status == EndpointHealthStatus.DEGRADED finally: - backend.shutdown() + await backend.shutdown() # ================================================================== # @@ -366,7 +366,7 @@ async def test_poller_receives_health_updates(self, mock_latency_service): class TestLatencyServiceBackendReadiness: """Verify is_ready() behavior with a real poller.""" - def test_ready_after_first_poll(self, mock_latency_service): + async def test_ready_after_first_poll(self, mock_latency_service): """is_ready() should return True once the poller has fetched health.""" mock_latency_service.set_health(MODEL_A, "healthy") backend = _make_backend(mock_latency_service.url, [MODEL_A]) @@ -375,7 +375,7 @@ def test_ready_after_first_poll(self, mock_latency_service): _wait_for_first_poll(backend) assert backend.is_ready() finally: - backend.shutdown() + await backend.shutdown() # ================================================================== # @@ -424,4 +424,4 @@ async def test_conversation_sticks_to_one_endpoint(self, mock_latency_service): ) assert selected[0] in {MODEL_A, MODEL_B} finally: - backend.shutdown() + await backend.shutdown() diff --git a/tests/test_latency_service_health_metrics.py b/tests/test_latency_service_health_metrics.py index f4f9cdcf..9329137e 100644 --- a/tests/test_latency_service_health_metrics.py +++ b/tests/test_latency_service_health_metrics.py @@ -185,20 +185,20 @@ def test_success_after_prior_failure_emits_ok(self) -> None: class TestEmitterLifecycle: - def test_construction_registers_emitter(self) -> None: + async def test_construction_registers_emitter(self) -> None: assert prometheus_emitter._EMITTERS == [] backend = _make_backend(_config("model-A")) assert len(prometheus_emitter._EMITTERS) == 1 # Confirm table output includes this backend's lines. assert "switchyard_endpoint_status" in prometheus_emitter.render() - backend.shutdown() + await backend.shutdown() assert prometheus_emitter._EMITTERS == [] - def test_shutdown_idempotent(self) -> None: + async def test_shutdown_idempotent(self) -> None: """Lifespan tear-down may call shutdown more than once.""" backend = _make_backend(_config("model-A")) - backend.shutdown() - backend.shutdown() + await backend.shutdown() + await backend.shutdown() assert prometheus_emitter._EMITTERS == [] diff --git a/tests/test_latency_service_llm_backend.py b/tests/test_latency_service_llm_backend.py index bae6d7e7..f22f2617 100644 --- a/tests/test_latency_service_llm_backend.py +++ b/tests/test_latency_service_llm_backend.py @@ -596,8 +596,8 @@ async def test_distinct_conversations_pin_independently(self): # Independent pins: each conversation resolves to its own endpoint. assert len(backend._affinity) == 2 - assert backend._affinity.pinned(ProxyContext(), req_a) == "model-A" - assert backend._affinity.pinned(ProxyContext(), req_b) == "model-B" + assert await backend._affinity.pinned(ProxyContext(), req_a) == "model-A" + assert await backend._affinity.pinned(ProxyContext(), req_b) == "model-B" async def test_degraded_pin_reroutes_and_repins(self): """When the pinned endpoint degrades, the next turn re-routes to a @@ -613,7 +613,7 @@ async def test_degraded_pin_reroutes_and_repins(self): }) req = _conv_request("task") await backend.call(ProxyContext(), req) - assert backend._affinity.pinned(ProxyContext(), req) == "model-A" + assert await backend._affinity.pinned(ProxyContext(), req) == "model-A" # model-A degrades; model-B becomes the only healthy endpoint. _set_health(backend, { @@ -623,7 +623,7 @@ async def test_degraded_pin_reroutes_and_repins(self): ctx2 = ProxyContext() await backend.call(ctx2, req) assert ctx2.selected_model == "model-B" - assert backend._affinity.pinned(ProxyContext(), req) == "model-B" + assert await backend._affinity.pinned(ProxyContext(), req) == "model-B" async def test_pin_follows_recovery_after_call_failure(self): """If the first-turn endpoint fails the call, the pin records the @@ -647,7 +647,7 @@ async def test_pin_follows_recovery_after_call_failure(self): await backend.call(ctx, req) # Regardless of which endpoint was tried first, only model-B succeeds. assert ctx.selected_model == "model-B" - assert backend._affinity.pinned(ProxyContext(), req) == "model-B" + assert await backend._affinity.pinned(ProxyContext(), req) == "model-B" async def test_lru_eviction_bounds_map(self): """The affinity map never exceeds affinity_max_sessions.""" @@ -709,6 +709,48 @@ async def test_affinity_counters_absent_when_disabled(self): assert "switchyard_affinity_hits_total" not in out assert "switchyard_affinity_misses_total" not in out + def test_affinity_l2_counters_rendered_when_shared_store_configured(self): + """L2 hit/error counters appear on /metrics when a shared store is on. + + Constructing the Redis store never connects (the client is lazy), so + this stays hermetic. + """ + backend = _make_backend(_config( + "model-A", + session_affinity=True, + affinity_store="redis", + affinity_store_url="redis://cache.test:6379/0", + )) + + out = "\n".join(backend._render_prometheus_lines()) + assert "switchyard_affinity_l2_hits_total 0" in out + assert "switchyard_affinity_l2_errors_total 0" in out + assert "switchyard_affinity_l2_breaker_open 0" in out + + def test_affinity_l2_breaker_gauge_reads_1_while_open(self): + """The breaker gauge flips to 1 once the failure streak opens it.""" + backend = _make_backend(_config( + "model-A", + session_affinity=True, + affinity_store="redis", + affinity_store_url="redis://cache.test:6379/0", + )) + affinity = backend._affinity + for _ in range(affinity._l2_breaker_threshold): + affinity._record_l2_failure("test-induced failure") + + out = "\n".join(backend._render_prometheus_lines()) + assert "switchyard_affinity_l2_breaker_open 1" in out + + def test_affinity_l2_counters_absent_for_memory_store(self): + """Without a shared store the L2 counters stay off the metric surface.""" + backend = _make_backend(_config("model-A", session_affinity=True)) + + out = "\n".join(backend._render_prometheus_lines()) + assert "switchyard_affinity_l2_hits_total" not in out + assert "switchyard_affinity_l2_errors_total" not in out + assert "switchyard_affinity_l2_breaker_open" not in out + def test_negative_affinity_max_rejected_at_construction(self): """A negative cap is rejected when the config is built — otherwise the LRU eviction loop in SessionCache.put would pop past an empty map.""" @@ -1556,11 +1598,34 @@ def test_ready_after_first_poll(self): backend._poller._poll_count = 1 assert backend.is_ready() is True - def test_shutdown_stops_poller(self): + async def test_shutdown_stops_poller(self): backend = _make_backend(_config("model-A")) - backend.shutdown() + await backend.shutdown() assert backend._poller._stop_event.is_set() + async def test_shutdown_closes_shared_affinity_store(self): + """Backend teardown releases the shared (L2) pin store's connections.""" + + class _ClosableL2: + def __init__(self) -> None: + self.closed = False + + async def get(self, key: str) -> str | None: + return None + + async def put(self, key: str, value: str) -> None: + return None + + async def aclose(self) -> None: + self.closed = True + + backend = _make_backend(_config("model-A", session_affinity=True)) + l2 = _ClosableL2() + backend._affinity._l2 = l2 + + await backend.shutdown() + assert l2.closed is True + # --------------------------------------------------------------------------- # Health poller diff --git a/tests/test_llm_classifier_request_processor.py b/tests/test_llm_classifier_request_processor.py index d8e47fab..57e82bb8 100644 --- a/tests/test_llm_classifier_request_processor.py +++ b/tests/test_llm_classifier_request_processor.py @@ -141,7 +141,7 @@ async def test_classifier_skips_llm_call_when_session_pinned() -> None: assert len(fake.calls) == 1 # The tier selector would pin the conversation after the first turn. - affinity.pin(ProxyContext(), req, "weak") + await affinity.pin(ProxyContext(), req, "weak") # Later turns of the same conversation skip the LLM call entirely. ctx2 = ProxyContext() @@ -163,15 +163,15 @@ async def test_classifier_runs_until_warmup_pin_exists() -> None: req1 = _request() await processor.process(ProxyContext(), req1) - affinity.pin(ProxyContext(), req1, "weak") + await affinity.pin(ProxyContext(), req1, "weak") req2 = _request(messages=_messages_with_prior_assistant_turns(1)) await processor.process(ProxyContext(), req2) - affinity.pin(ProxyContext(), req2, "weak") + await affinity.pin(ProxyContext(), req2, "weak") req3 = _request(messages=_messages_with_prior_assistant_turns(2)) await processor.process(ProxyContext(), req3) - affinity.pin(ProxyContext(), req3, "weak") + await affinity.pin(ProxyContext(), req3, "weak") assert len(fake.calls) == 3 @@ -193,7 +193,7 @@ async def test_classifier_runs_when_affinity_disabled() -> None: affinity=affinity, ) req = _request() - affinity.pin(ProxyContext(), req, "weak") # no-op while disabled + await affinity.pin(ProxyContext(), req, "weak") # no-op while disabled await processor.process(ProxyContext(), req) await processor.process(ProxyContext(), req) diff --git a/tests/test_redis_pin_store.py b/tests/test_redis_pin_store.py new file mode 100644 index 00000000..49a7269d --- /dev/null +++ b/tests/test_redis_pin_store.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for RedisPinStore adapter mechanics and its config → backend wiring. + +An injected fake async client keeps these hermetic — no live Redis and no +``fakeredis`` dependency. Building the L2 exercises the lazy-import path without +requiring ``redis`` to be installed, because the real client is only created on +first ``get``/``put``. +""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from switchyard.lib.backends.latency_service_llm_backend import _build_affinity_l2 +from switchyard.lib.config.latency_service_backend_config import ( + LatencyServiceBackendConfig, + LatencyServiceEndpoint, +) +from switchyard.lib.redis_pin_store import RedisPinStore + + +class _FakeAsyncRedis: + """Minimal async Redis double capturing set() arguments.""" + + def __init__(self) -> None: + self.kv: dict[str, str] = {} + self.set_calls: list[tuple[str, str, int | None]] = [] + + async def get(self, key: str) -> str | None: + return self.kv.get(key) + + async def set(self, key: str, value: str, ex: int | None = None) -> None: + self.set_calls.append((key, value, ex)) + self.kv[key] = value + + +# --- adapter mechanics ------------------------------------------------------ + + +async def test_put_writes_prefixed_key_with_ttl() -> None: + fake = _FakeAsyncRedis() + store = RedisPinStore("redis://x", ttl_seconds=123, key_prefix="p:", client=fake) + await store.put("abc", "model-A") + assert fake.set_calls == [("p:abc", "model-A", 123)] + + +async def test_get_reads_prefixed_key_and_roundtrips() -> None: + fake = _FakeAsyncRedis() + store = RedisPinStore("redis://x", key_prefix="p:", client=fake) + assert await store.get("abc") is None + await store.put("abc", "model-B") + assert await store.get("abc") == "model-B" + + +def test_invalid_ttl_rejected() -> None: + with pytest.raises(ValueError): + RedisPinStore("redis://x", ttl_seconds=0) + + +async def test_aclose_closes_client_once() -> None: + """aclose releases the client's pool; repeated calls don't double-close.""" + + class _ClosableFake(_FakeAsyncRedis): + def __init__(self) -> None: + super().__init__() + self.closes = 0 + + async def aclose(self) -> None: + self.closes += 1 + + fake = _ClosableFake() + store = RedisPinStore("redis://x", client=fake) + await store.aclose() + await store.aclose() # client reference already released → no-op + assert fake.closes == 1 + + +async def test_aclose_before_first_use_is_noop() -> None: + """Closing a store that never built a client must not import or connect.""" + await RedisPinStore("redis://x").aclose() + + +# --- config validation + backend wiring ------------------------------------- + + +def _redis_config(**overrides: object) -> LatencyServiceBackendConfig: + base: dict[str, object] = { + "endpoints": [LatencyServiceEndpoint(model="m")], + "session_affinity": True, + "affinity_store": "redis", + "affinity_store_url": "redis://cache:6379/0", + } + base.update(overrides) + return LatencyServiceBackendConfig(**base) # type: ignore[arg-type] + + +def test_redis_store_requires_url() -> None: + with pytest.raises(ValidationError): + _redis_config(affinity_store_url=None) + + +def test_redis_store_requires_session_affinity() -> None: + with pytest.raises(ValidationError): + _redis_config(session_affinity=False) + + +def test_memory_is_the_default_store() -> None: + cfg = LatencyServiceBackendConfig(endpoints=[LatencyServiceEndpoint(model="m")]) + assert cfg.affinity_store == "memory" + assert _build_affinity_l2(cfg) is None + + +def test_build_affinity_l2_threads_config_into_redis_store() -> None: + cfg = _redis_config(affinity_store_ttl_seconds=42, affinity_key_prefix="k:") + store = _build_affinity_l2(cfg) + assert isinstance(store, RedisPinStore) + assert store._ttl_seconds == 42 + assert store._key_prefix == "k:" diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py index 4bbc7bd0..5544e97b 100644 --- a/tests/test_route_bundle.py +++ b/tests/test_route_bundle.py @@ -1673,6 +1673,37 @@ def test_zero_capacity_affinity_rejected_via_bundle(route: dict[str, Any]) -> No build_route_bundle_table({"routes": {"r": {**route, "affinity_max_sessions": 0}}}) +def test_latency_affinity_redis_reaches_backend_via_bundle() -> None: + """Redis L2 keys parse and construct a RedisPinStore on the latency backend.""" + from switchyard.lib.redis_pin_store import RedisPinStore + + table = build_route_bundle_table({ + "routes": { + "r": { + **_AFFINITY_LAT_ROUTE, + "affinity_store": "redis", + "affinity_store_url": "redis://cache:6379/0", + "affinity_store_ttl_seconds": 120, + "affinity_key_prefix": "k:", + }, + }, + }) + backend = _latency_backend(table.lookup_switchyard("r")) + assert backend._config.affinity_store == "redis" + assert backend._config.affinity_store_url == "redis://cache:6379/0" + assert isinstance(backend._affinity._l2, RedisPinStore) + + +def test_latency_affinity_redis_requires_url_via_bundle() -> None: + """affinity_store=redis without a URL fails closed at config load.""" + from pydantic import ValidationError + + with pytest.raises(ValidationError): + build_route_bundle_table({ + "routes": {"r": {**_AFFINITY_LAT_ROUTE, "affinity_store": "redis"}}, + }) + + def test_negative_affinity_warmup_turns_rejected_via_bundle() -> None: """The deterministic config validates affinity_warmup_turns as non-negative.""" from pydantic import ValidationError diff --git a/tests/test_session_affinity.py b/tests/test_session_affinity.py new file mode 100644 index 00000000..26cd8897 --- /dev/null +++ b/tests/test_session_affinity.py @@ -0,0 +1,294 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for SessionAffinity's L1 (in-process) + optional L2 (shared) tiering. + +The latency backend and classifier router integration tests cover affinity +*policy*; these tests cover the store mechanics in isolation: write-through, +read-through with L1 warming, and best-effort (fail-open) degradation when the +L2 errors. +""" + +from __future__ import annotations + +from switchyard.lib.proxy_context import ProxyContext +from switchyard.lib.session_affinity import SessionAffinity +from switchyard_rust.core import ChatRequest + + +def _request(text: str = "hi") -> ChatRequest: + """A turn-1 request whose first user message anchors a distinct session.""" + return ChatRequest.openai_chat( + {"model": "incoming-model", "messages": [{"role": "user", "content": text}]} + ) + + +class _DictL2: + """In-memory AffinityPinStore double with call counters.""" + + def __init__(self) -> None: + self.store: dict[str, str] = {} + self.gets = 0 + self.puts = 0 + + async def get(self, key: str) -> str | None: + self.gets += 1 + return self.store.get(key) + + async def put(self, key: str, value: str) -> None: + self.puts += 1 + self.store[key] = value + + +class _BrokenL2: + """AffinityPinStore double that always raises, to exercise fail-open.""" + + async def get(self, key: str) -> str | None: + raise RuntimeError("l2 unavailable") + + async def put(self, key: str, value: str) -> None: + raise RuntimeError("l2 unavailable") + + +async def test_no_l2_still_pins_via_l1() -> None: + """The async interface works L1-only when no L2 is configured.""" + affinity = SessionAffinity(enabled=True) + req = _request("task") + await affinity.pin(ProxyContext(), req, "model-A") + assert await affinity.pinned(ProxyContext(), req) == "model-A" + + +async def test_pin_writes_through_to_l2() -> None: + """A pin lands in both L1 and the shared L2.""" + l2 = _DictL2() + affinity = SessionAffinity(enabled=True, l2=l2) + req = _request("task") + await affinity.pin(ProxyContext(), req, "model-A") + assert l2.puts == 1 + assert list(l2.store.values()) == ["model-A"] + + +async def test_l1_miss_reads_through_l2_and_warms_l1() -> None: + """A pin written by one worker is visible to another via the shared L2, + and the read-through warms the reader's L1 so later turns skip the round-trip.""" + l2 = _DictL2() + worker_a = SessionAffinity(enabled=True, l2=l2) + worker_b = SessionAffinity(enabled=True, l2=l2) + req = _request("shared task") + + # Worker A serves the first turn and pins. + await worker_a.pin(ProxyContext(), req, "model-A") + + # Worker B has never seen this conversation: L1 miss → L2 read-through. + assert await worker_b.pinned(ProxyContext(), req) == "model-A" + assert l2.gets == 1 + assert worker_b.l2_hits == 1 + + # L1 is now warm on worker B: a second lookup does not hit L2 again. + assert await worker_b.pinned(ProxyContext(), req) == "model-A" + assert l2.gets == 1 + assert worker_b.l2_hits == 1 + + +async def test_l2_get_failure_is_fail_open() -> None: + """An L2 read error routes as unpinned rather than raising, and is counted.""" + affinity = SessionAffinity(enabled=True, l2=_BrokenL2()) + assert await affinity.pinned(ProxyContext(), _request("task")) is None + assert affinity.l2_errors == 1 + + +async def test_l2_put_failure_is_fail_open_and_keeps_l1_pin() -> None: + """An L2 write error does not raise; the pin still holds in the local L1.""" + affinity = SessionAffinity(enabled=True, l2=_BrokenL2()) + req = _request("task") + await affinity.pin(ProxyContext(), req, "model-A") # must not raise + assert affinity.l2_errors == 1 + # L1 retained the pin, so a same-worker lookup still resolves (no L2 read). + assert await affinity.pinned(ProxyContext(), req) == "model-A" + assert affinity.l2_errors == 1 # the L1 hit never consulted the broken L2 + + +async def test_disabled_never_touches_l2() -> None: + """A disabled instance no-ops without reading or writing the shared store.""" + l2 = _DictL2() + affinity = SessionAffinity(enabled=False, l2=l2) + req = _request("task") + await affinity.pin(ProxyContext(), req, "model-A") + assert await affinity.pinned(ProxyContext(), req) is None + assert l2.gets == 0 + assert l2.puts == 0 + + +async def test_warmup_turn_never_touches_l2() -> None: + """Turns inside the warmup window neither read nor write the shared store.""" + l2 = _DictL2() + affinity = SessionAffinity(enabled=True, warmup_turns=1, l2=l2) + req = _request("task") # turn 1, within warmup_turns=1 + await affinity.pin(ProxyContext(), req, "model-A") + assert await affinity.pinned(ProxyContext(), req) is None + assert l2.gets == 0 + assert l2.puts == 0 + + +async def test_aclose_delegates_to_l2() -> None: + """Closing the affinity coordinator releases the shared store.""" + + class _ClosableL2(_DictL2): + def __init__(self) -> None: + super().__init__() + self.closed = False + + async def aclose(self) -> None: + self.closed = True + + l2 = _ClosableL2() + affinity = SessionAffinity(enabled=True, l2=l2) + await affinity.aclose() + assert l2.closed + + +async def test_aclose_is_noop_without_l2_or_without_aclose() -> None: + """L1-only instances and L2s lacking aclose() close without error.""" + await SessionAffinity(enabled=True).aclose() + await SessionAffinity(enabled=True, l2=_DictL2()).aclose() # no aclose method + + +async def test_aclose_swallows_l2_close_errors() -> None: + """A failing store close is logged, never raised — teardown can't wedge.""" + + class _BrokenClose(_DictL2): + async def aclose(self) -> None: + raise RuntimeError("close failed") + + await SessionAffinity(enabled=True, l2=_BrokenClose()).aclose() # must not raise + + +# --------------------------------------------------------------------------- +# L2 circuit breaker +# --------------------------------------------------------------------------- + + +class _Clock: + """Manually-advanced monotonic clock for deterministic breaker tests.""" + + def __init__(self) -> None: + self.now = 0.0 + + def __call__(self) -> float: + return self.now + + +class _FlakyL2(_BrokenL2): + """Failing store that counts attempts and can be healed mid-test.""" + + def __init__(self) -> None: + self.attempts = 0 + self.healed = False + self.store: dict[str, str] = {} + + async def get(self, key: str) -> str | None: + self.attempts += 1 + if not self.healed: + raise RuntimeError("l2 unavailable") + return self.store.get(key) + + async def put(self, key: str, value: str) -> None: + self.attempts += 1 + if not self.healed: + raise RuntimeError("l2 unavailable") + self.store[key] = value + + +def _breaker_affinity(l2: _FlakyL2, clock: _Clock) -> SessionAffinity: + return SessionAffinity( + enabled=True, + l2=l2, + l2_breaker_threshold=3, + l2_breaker_cooldown_s=10.0, + clock=clock, + ) + + +async def test_l2_breaker_opens_after_streak_and_skips_operations() -> None: + """Threshold consecutive failures open the breaker; further operations + never reach the store while the cooldown runs.""" + l2, clock = _FlakyL2(), _Clock() + affinity = _breaker_affinity(l2, clock) + + for i in range(3): + await affinity.pin(ProxyContext(), _request(f"t{i}"), "model-A") + assert l2.attempts == 3 + assert affinity.l2_breaker_open + + # Open breaker: neither reads (distinct sessions = L1 misses) nor writes + # touch the store. + assert await affinity.pinned(ProxyContext(), _request("t-new")) is None + await affinity.pin(ProxyContext(), _request("t-newer"), "model-A") + assert l2.attempts == 3 + assert affinity.l2_errors == 3 + + +async def test_l2_breaker_probes_after_cooldown_and_closes_on_success() -> None: + """The first operation after the cooldown is the probe; success closes the + breaker and normal read/write-through resumes.""" + l2, clock = _FlakyL2(), _Clock() + affinity = _breaker_affinity(l2, clock) + + for i in range(3): + await affinity.pin(ProxyContext(), _request(f"t{i}"), "model-A") + l2.healed = True + clock.now = 10.0 # cooldown elapsed → next op is the probe + + await affinity.pin(ProxyContext(), _request("probe"), "model-A") + + assert not affinity.l2_breaker_open + assert l2.attempts == 4 + # Fully closed: later operations go through again. + await affinity.pin(ProxyContext(), _request("after"), "model-A") + assert l2.attempts == 5 + + +async def test_l2_breaker_failed_probe_rearms_the_cooldown() -> None: + """A failing probe re-opens the breaker for another full cooldown.""" + l2, clock = _FlakyL2(), _Clock() + affinity = _breaker_affinity(l2, clock) + + for i in range(3): + await affinity.pin(ProxyContext(), _request(f"t{i}"), "model-A") + clock.now = 10.0 # still broken: the probe fails + + await affinity.pin(ProxyContext(), _request("probe"), "model-A") + + assert l2.attempts == 4 + assert affinity.l2_breaker_open + # Mid-cooldown operations stay skipped. + clock.now = 19.9 + await affinity.pin(ProxyContext(), _request("mid"), "model-A") + assert l2.attempts == 4 + + +async def test_l2_non_consecutive_failures_do_not_open_the_breaker() -> None: + """A success resets the streak: intermittent blips never trip the breaker.""" + l2, clock = _FlakyL2(), _Clock() + affinity = _breaker_affinity(l2, clock) + + await affinity.pin(ProxyContext(), _request("t0"), "model-A") + await affinity.pin(ProxyContext(), _request("t1"), "model-A") + l2.healed = True + await affinity.pin(ProxyContext(), _request("t2"), "model-A") # success resets + l2.healed = False + await affinity.pin(ProxyContext(), _request("t3"), "model-A") + await affinity.pin(ProxyContext(), _request("t4"), "model-A") + + assert not affinity.l2_breaker_open + assert affinity.l2_errors == 4 + + +async def test_l2_breaker_config_validated() -> None: + """Breaker knobs reject nonsensical values.""" + import pytest + + with pytest.raises(ValueError): + SessionAffinity(enabled=True, l2_breaker_threshold=0) + with pytest.raises(ValueError): + SessionAffinity(enabled=True, l2_breaker_cooldown_s=0) diff --git a/uv.lock b/uv.lock index b323f3b0..8594e3b7 100644 --- a/uv.lock +++ b/uv.lock @@ -1174,9 +1174,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/7a/6bc2a7835731387ed303b9390ce68a116ab053df05450a59181239200454/greenlet-3.5.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:76dae33e97b52743a19210931ee3e78a88fe1438bc2fc4ee5e7512d289bfad4f", size = 288351, upload-time = "2026-06-17T17:36:17.019Z" }, { url = "https://files.pythonhosted.org/packages/57/1b/bd98062fcef6d0e9d0873ab6f2d029772e6ea342972ae43275bd6177900f/greenlet-3.5.2-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30252d191d6959df1d040b559a38fc017139606c5ecc2ad00416557c0355d742", size = 604273, upload-time = "2026-06-17T18:07:20.296Z" }, { url = "https://files.pythonhosted.org/packages/25/e6/fe392c522bf45d976abe7db2793f6ef4e87b053ebb869deeaae46aeb54da/greenlet-3.5.2-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1adc23c50f22b0f5979521909a8360ab4a3d3bef8b641ce633a04cf1b1c967ea", size = 616536, upload-time = "2026-06-17T18:29:43.205Z" }, - { url = "https://files.pythonhosted.org/packages/42/df/cdb1f75f07214f13110e7e3879531f11c26083bd480a56a9474c430ec44c/greenlet-3.5.2-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87359c23eb4e8f1b16da68faad29bf5aeb80e3628d7d8e4aa2e41c36879ddedd", size = 621843, upload-time = "2026-06-17T18:39:27.507Z" }, { url = "https://files.pythonhosted.org/packages/68/4a/399ff81fa93a19d6a9df394cef0355f082dbc19ad41aba9593cd0ad444e2/greenlet-3.5.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f052fff492c52fdfa99bd3b3c1389a53de37dae76a0562741417f0d018f02b3", size = 613749, upload-time = "2026-06-17T17:39:28.148Z" }, - { url = "https://files.pythonhosted.org/packages/2e/25/36a3628a7edcfeefddd3101dc88039c79721c5f8d688db7ebed1cbaaa789/greenlet-3.5.2-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:f4d67c1684db3f9782c37ee4bade3f86f5a23a8fcf3f8359224106018ca40728", size = 424889, upload-time = "2026-06-17T18:41:19.469Z" }, { url = "https://files.pythonhosted.org/packages/a5/75/f519593f12ad43d08e28c03a95cfe2eeae011707dbc9dab0c4a263ce90f9/greenlet-3.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:120b77c2a18ebf629c3a7886f68c6d01e065654844ad468f15bb93ace66f2094", size = 1573725, upload-time = "2026-06-17T18:22:12.023Z" }, { url = "https://files.pythonhosted.org/packages/f1/bc/bc1ea4b0754c6c51bbf9d94677b0b1f7fbda8cbb404e44a896854fc0a940/greenlet-3.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a850f6224088ef7dcc70f1a545cb6b3d119c35d6dca63b925b9f35da0635cdad", size = 1638132, upload-time = "2026-06-17T17:40:06.971Z" }, { url = "https://files.pythonhosted.org/packages/36/c0/f0f5a34247df60de285f75f22e57f14027f4b3c43820981854b5b643ca6d/greenlet-3.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:89da99ee8345b458ea2f16831dad31c88ddcdec454b48704d569a0b8fb28f146", size = 239393, upload-time = "2026-06-17T17:33:47.09Z" }, @@ -1184,9 +1182,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/3c/bb37b9d40d65b0741a8b040ca5c307034d0a9822994dff5f825c88dd7a6b/greenlet-3.5.2-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:0629377725977252159de1ebd3c6e49c170a63856e585446797bb3d66d4d9c34", size = 287178, upload-time = "2026-06-17T17:35:25.132Z" }, { url = "https://files.pythonhosted.org/packages/f0/a6/0c5902393f492f8ceb19d0b5cf139284e3a11b333a049739643b1036b6f8/greenlet-3.5.2-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2ddf9eddc617681108dd071b3feabf3f4a4cd64846254aec4d4ceda098b639a", size = 606900, upload-time = "2026-06-17T18:07:21.692Z" }, { url = "https://files.pythonhosted.org/packages/d8/7c/42899c31d4b87148ae4e3f87f63e13398824be6241f4dde42ded95768a34/greenlet-3.5.2-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f41feb9f2b59e2e61ac9bea4e344ddd9396bf3cacb2583f73a3595ed7df6f8e7", size = 619265, upload-time = "2026-06-17T18:29:44.837Z" }, - { url = "https://files.pythonhosted.org/packages/6a/7e/28f991affb413b232b1e7d768db24c37b3f4d5daecc3f19b455d40bd2dea/greenlet-3.5.2-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9dc23f0e5ad76415457212a4b947d22ebe4dc80baf02adf7dd5647a90f38bb4e", size = 625044, upload-time = "2026-06-17T18:39:29.046Z" }, { url = "https://files.pythonhosted.org/packages/d3/52/4ff8c98d3cfe62b4515f8584ae14510a58f35c549cc5292b78d9b7a40b70/greenlet-3.5.2-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09201fa698768db245920b00fdc86ee3e73540f01ca6db162be9632642e1a473", size = 616187, upload-time = "2026-06-17T17:39:29.473Z" }, - { url = "https://files.pythonhosted.org/packages/29/05/0cc9ec660e7acff85f93b0a048b6654371c822c884add44c02a465cf70e0/greenlet-3.5.2-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:423167363c510a75b649f5cd58d873c29498ea03598b9e4b1c3b73e0f899f3d5", size = 427322, upload-time = "2026-06-17T18:41:20.892Z" }, { url = "https://files.pythonhosted.org/packages/c9/a6/269c8bf9aefc13361ce1088f0e392b154cb21005de7862e42b5d782b81fd/greenlet-3.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a1759fa4f14c398508cf20dc8037de55cc23ae8bd14c185c2718257837195ca5", size = 1573778, upload-time = "2026-06-17T18:22:13.497Z" }, { url = "https://files.pythonhosted.org/packages/1f/9b/391d015cbc6323e81b14c02cf825fdca7e0049c9bb489bf4ac72883118ba/greenlet-3.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b9318cdeb9abdbfdd8bc8464ee4a06dffde2c7846e1def138365a6240ab2c9a5", size = 1638092, upload-time = "2026-06-17T17:40:08.163Z" }, { url = "https://files.pythonhosted.org/packages/49/53/5b4df711f4356c62e85d9f819d87966d526d1cfb32bae49a8f7d6fc36ea4/greenlet-3.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:2c3b3311af72b3d3b03cc0f1ffd11f072e834be5d0444105cf715fc44434e39c", size = 239352, upload-time = "2026-06-17T17:38:51.593Z" }, @@ -1194,9 +1190,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/89/aaafc8e14de4ac882e02ccb963225329b0e8578aba4365e71eb678e45722/greenlet-3.5.2-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:1c31219badba285858ba8ed117f403dea7fafee6bade9a1991875aae530c3ceb", size = 287676, upload-time = "2026-06-17T17:33:31.514Z" }, { url = "https://files.pythonhosted.org/packages/b8/fc/2308249206c12ac70de7b9a00970f84f07d10b3cd60e05d2fbcaa84124e8/greenlet-3.5.2-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6f96ed6f4adc1066954ae95f45717657cb67468ef3b89e9a3632e14a625a8f39", size = 653552, upload-time = "2026-06-17T18:07:23.493Z" }, { url = "https://files.pythonhosted.org/packages/7c/24/47730d1f8f1336b9b089237521ed7a26eee997065dcb4cab81cdca333abc/greenlet-3.5.2-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5795e883e915333c0d5648faaa691857fbc7180136883edc377f50f0d509c2a8", size = 665756, upload-time = "2026-06-17T18:29:46.616Z" }, - { url = "https://files.pythonhosted.org/packages/23/5c/2664d290cbd1fef9eb3f69b5d3bc5aa91b6fa907519298ca6af93a90c6cb/greenlet-3.5.2-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e9e49d732ee92a189bb7035e293029244aeba648297a9b856dc733d17ca7f0d", size = 669989, upload-time = "2026-06-17T18:39:30.79Z" }, { url = "https://files.pythonhosted.org/packages/99/69/d6c99db15dc0b5e892ac3cc7b942c8b21f4a9cc3bd9ea0bc3b0f339ffbd4/greenlet-3.5.2-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26aed8d9503ca78889141a9739d71b383efea5f472a7c522b5410f7eb2a1b163", size = 663228, upload-time = "2026-06-17T17:39:31.073Z" }, - { url = "https://files.pythonhosted.org/packages/42/d4/fcb53fa9847d7fbd4723fbed9469c3869b9e3544c4e001d9d5aa2f66162d/greenlet-3.5.2-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:537c5c4f30395020bb9f48f53146070e3b997c3c75da14011ab732aaa19ce3ef", size = 472888, upload-time = "2026-06-17T18:41:22.511Z" }, { url = "https://files.pythonhosted.org/packages/4f/88/9e603f448e2bc107c883e95817b980fb9b45ba6aea0299b2e9978124bea2/greenlet-3.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:dbebc038fcdda8f8f21cce985fd04e34e0f42007e7fc7ab7ad285caf77974b95", size = 1620723, upload-time = "2026-06-17T18:22:14.817Z" }, { url = "https://files.pythonhosted.org/packages/11/91/26da17e3777858c16fdb8d020a4c68f3a03cb92f238de8f5351d5d5186e9/greenlet-3.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a207023f1cf8695fd82580b8099c09c5809be18bc2282362cdfb965dd884a317", size = 1684227, upload-time = "2026-06-17T17:40:09.536Z" }, { url = "https://files.pythonhosted.org/packages/2d/44/b3a11f7aa34cb38f1b7f3df8bcd9fcd09bac9d342c2a2c9b8686c804bcd2/greenlet-3.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:c674a1dd4fe41f6a93febe7ab366ceabf15080ea31a9307811c56dac5f435f73", size = 240257, upload-time = "2026-06-17T17:35:23.359Z" }, @@ -1204,18 +1198,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/47/ac/d3bad483e9f6cd1848604fdffa32cac25846dd6dfcec0e6f81c790185518/greenlet-3.5.2-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:a96457a30384de52d9c5d2fd33abf6c1daae3db392cd556738f408b1a79a1cf0", size = 295668, upload-time = "2026-06-17T17:36:02.293Z" }, { url = "https://files.pythonhosted.org/packages/00/e9/3a7e557b895fd0469b00cd0b2bd498ba950e8bfdf6d7adeecf2c5e4130a6/greenlet-3.5.2-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e4af5d4961818ab651d09c1448a03b1ba2a1726a076266ebb62330bab9f3238c", size = 652820, upload-time = "2026-06-17T18:07:24.95Z" }, { url = "https://files.pythonhosted.org/packages/78/67/6225d5c5e4afc04be0fd161eec82e4b72017e8a100d222f25d7b42b0140d/greenlet-3.5.2-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a1789a6244ea1ba61fd4386c9a6a31873e9b0234762103364be98ef87dcb19f3", size = 658697, upload-time = "2026-06-17T18:29:48.365Z" }, - { url = "https://files.pythonhosted.org/packages/35/ad/9b3058f999b81750a9c6d9ec424f509462d232b58002086fe2ba63b66407/greenlet-3.5.2-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2ee6288f1933d698b4f098127ed17bda2910a75d2807915bd16294a972055d6c", size = 658945, upload-time = "2026-06-17T18:39:32.509Z" }, { url = "https://files.pythonhosted.org/packages/fa/99/6324b8ef916dcaddccb340b304c992ca3f947614ce0f2685d438187300b8/greenlet-3.5.2-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3be00501fb4a8c37f6b4b3c4773808ceb26ea65c7ea64fd5735d0f330b3786de", size = 656436, upload-time = "2026-06-17T17:39:32.509Z" }, - { url = "https://files.pythonhosted.org/packages/92/75/1b6ecd8c027b69ab1b6798a84094df79aab5e69ac7e249c78b9d361dd1fa/greenlet-3.5.2-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:b4cad42662c796334c2d24607c411e3ed82481c1fb4e1e8ec3a5a8416060092e", size = 490529, upload-time = "2026-06-17T18:41:23.954Z" }, { url = "https://files.pythonhosted.org/packages/a9/ee/f5bf9daac27c5e1b011965f64b5630a32b415daf7381b312943629e12c2a/greenlet-3.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1d554cd96841a68d464d75a3736f8e87408a7b02b1930a75fa32feb408ad62f8", size = 1617193, upload-time = "2026-06-17T18:22:16.252Z" }, { url = "https://files.pythonhosted.org/packages/8a/21/b05d5b12715bda92ce27c118d64971d21e9b8f3563ed959a7d271e2d4223/greenlet-3.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3dff6cd3aac35f6cd3fc23460105acf576f5faf6c378de0bc088bf37c913864a", size = 1677512, upload-time = "2026-06-17T17:40:10.771Z" }, { url = "https://files.pythonhosted.org/packages/b8/97/1b8f1314b868041b327dc1051603e8142b826480cb0ecb8a7b7632aee9c4/greenlet-3.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:36cfea2aa075d544617176b2e84450480f0797070ad8799a8c41ada2fe449d32", size = 243145, upload-time = "2026-06-17T17:34:37.502Z" }, { url = "https://files.pythonhosted.org/packages/36/07/1b5311775e04c718a118c504d7a3a312430e2a1bd1347226aff4774e4549/greenlet-3.5.2-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:a0314aa832c94633355dc6f3ee54f195159533355a323f26926fc63b98b2ccbb", size = 288315, upload-time = "2026-06-17T17:34:34.04Z" }, { url = "https://files.pythonhosted.org/packages/ed/cc/6abcd2a486b58b9f77b7a93b690d59cb2c11a5906ed2ad4c63c7b9c1113d/greenlet-3.5.2-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:24c59cb7db9d5c694cb8fd0c76eef8e456b2123afdfa7e4b8f2a67a0860d7682", size = 659130, upload-time = "2026-06-17T18:07:26.354Z" }, { url = "https://files.pythonhosted.org/packages/f2/12/f4aaad6d3d383233f700ab322568a4f29f2c701a4861d85f4811d99689b2/greenlet-3.5.2-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7bb811753703739ad318112f16eccfaabdac050037b6d092debaa8b23566b4ce", size = 669724, upload-time = "2026-06-17T18:29:50.13Z" }, - { url = "https://files.pythonhosted.org/packages/53/e0/4ce3a046b51e53934eae93d7f9c13975a97285741e9e1fcadf8751314c37/greenlet-3.5.2-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2debcd0ef9455b7d4879589903efc8e497d4b8fb8c0ae772309e44d1ca5e957f", size = 673494, upload-time = "2026-06-17T18:39:34.196Z" }, { url = "https://files.pythonhosted.org/packages/91/2a/a089811fc31c6bf8742f40a4e73470d6d401cef18e4314eb20dc399b377c/greenlet-3.5.2-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6d78b5c1c178dad90447f1b8452262709d3eef4c98f825569e74c9d0b2260ac9", size = 668089, upload-time = "2026-06-17T17:39:33.808Z" }, - { url = "https://files.pythonhosted.org/packages/52/e0/9c18721e63445dce02ee67e4c81c0f281626604ff55ae6f7b7f4354d7129/greenlet-3.5.2-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:9558cae989faeab6fbb425cd98a0cfa4190a47fba6443973fbee0a1eb0b0b6c3", size = 479721, upload-time = "2026-06-17T18:41:25.726Z" }, { url = "https://files.pythonhosted.org/packages/0f/1c/2f47c7d5fcfa98a62b705bf9a0505d86f4563c0d81cab1f7159ff1e743b7/greenlet-3.5.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:0977af2df83136f81c1f76e76d4e2fe7d0dc56ea9c101a86af26a95190b9ca32", size = 1625684, upload-time = "2026-06-17T18:22:17.664Z" }, { url = "https://files.pythonhosted.org/packages/b9/bf/661dd24624f70b7b32972d7693d0344ecde10278f647d7b828baf739899c/greenlet-3.5.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f9ed777c6891d8253e54468576f55e27f8fc1a662a664f946a191003574c0a74", size = 1688043, upload-time = "2026-06-17T17:40:12.403Z" }, { url = "https://files.pythonhosted.org/packages/60/49/d9bde1d15a21296b3b521fe083eb8aabd54ac05d15de9832918f3d639543/greenlet-3.5.2-cp315-cp315-win_amd64.whl", hash = "sha256:c0ea4eb3de23f0bac1d75205e10ccfa9b418b17b01a2d7bf19e3b69dda08900a", size = 240531, upload-time = "2026-06-17T17:35:47.448Z" }, @@ -1223,9 +1213,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/92/15/907be5e8900901039bae752fa9a31c03a3c1e064833f35a4e49449184581/greenlet-3.5.2-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:98a52d6a50d4deaba304331d83ee3e10ebbdc1517fcca40b2715d1de4534065c", size = 296697, upload-time = "2026-06-17T17:37:15.887Z" }, { url = "https://files.pythonhosted.org/packages/95/5c/08c57be575c3d6a3c023bbf22144a1c7dc6ed4d134527bb36ded4dbf04a8/greenlet-3.5.2-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1587ff8b58fdf806993ed1490a06ac19c22d47b219c68b30954380029045d8d4", size = 656710, upload-time = "2026-06-17T18:07:28.046Z" }, { url = "https://files.pythonhosted.org/packages/8c/d0/749f917bdc9fc90fceea4aa65fbf6556e617a50714d1496bdc8ad190bb36/greenlet-3.5.2-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:feb721811d2754bfd16b48de151dd6b1f222c048e625151f2ca44cfdfd69f59c", size = 662629, upload-time = "2026-06-17T18:29:51.728Z" }, - { url = "https://files.pythonhosted.org/packages/55/87/10776cd88df54d0f563e9e21e98363f2d6af94bedc553b1da0972fa87f80/greenlet-3.5.2-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9476cbead736dc48ce89e3cd97acff95ecc48cbf21273603a438f9870c4a014", size = 663191, upload-time = "2026-06-17T18:39:35.639Z" }, { url = "https://files.pythonhosted.org/packages/5a/a5/68cefae3a07f6d0093a490cf28ab604f14578f3e60205a2a2b2d5cd70af2/greenlet-3.5.2-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fe6062b1f35534e1e8fb28dfed406cf4eeff3e0bca3a0d9f8ff69f20a4abb00", size = 660147, upload-time = "2026-06-17T17:39:35.068Z" }, - { url = "https://files.pythonhosted.org/packages/02/aa/26ddf92826a99d87bfb8fdb8f3a262a6f16495a5d8e579737baa92fb4543/greenlet-3.5.2-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:5930d3946ecae99fa7fc0e3f3ae515426ad85058ebd9bfc6c00cca8016e6206b", size = 498199, upload-time = "2026-06-17T18:41:27.464Z" }, { url = "https://files.pythonhosted.org/packages/d2/6b/b9156d8397e4750220f54c7c5c34650f1e740a8d2f66eab9cfd1b7b53b69/greenlet-3.5.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:b4ac902af825cbac8e9b2fccab8122236fd2ba6c8b71a080116d2c2ec72671b1", size = 1621675, upload-time = "2026-06-17T18:22:18.873Z" }, { url = "https://files.pythonhosted.org/packages/b0/e3/d3250f4fa01c211a93d04e34fded63187e648dbec17b9b1a14d388040593/greenlet-3.5.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:6f1e473c06ae8be00c9034c2bb10fa277b08a93287e3111c395b839f01d27e1f", size = 1680577, upload-time = "2026-06-17T17:40:14.055Z" }, { url = "https://files.pythonhosted.org/packages/55/ba/eaee8bda4419770d7096b5a009ebff0ab20a2a28cdd83c4b591bfdf36fa9/greenlet-3.5.2-cp315-cp315t-win_amd64.whl", hash = "sha256:3c2315045f9983e2e50d7e89d95405c21bddb8745f2da4487bc080ab3525f904", size = 243482, upload-time = "2026-06-17T17:37:34.741Z" }, @@ -2295,11 +2283,15 @@ dependencies = [ ] [package.optional-dependencies] +affinity-redis = [ + { name = "redis" }, +] all = [ { name = "ddtrace" }, { name = "fastapi" }, { name = "nemo-platform" }, { name = "prompt-toolkit" }, + { name = "redis" }, { name = "routellm", extra = ["serve"] }, { name = "sse-starlette" }, { name = "transformers" }, @@ -2355,16 +2347,17 @@ requires-dist = [ { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.136.1,<1.0" }, { name = "httpx", specifier = ">=0.28.1,<1.0" }, { name = "nemo-platform", marker = "extra == 'intake'", specifier = ">=0.1.0,<1.0" }, - { name = "nemo-switchyard", extras = ["server", "cli", "gpu", "tracing", "intake"], marker = "extra == 'all'" }, + { name = "nemo-switchyard", extras = ["server", "cli", "gpu", "tracing", "intake", "affinity-redis"], marker = "extra == 'all'" }, { name = "openai", specifier = ">=2.34.0,<3.0" }, { name = "prompt-toolkit", marker = "extra == 'cli'", specifier = ">=3.0.52,<4.0" }, { name = "pydantic", specifier = ">=2.13.3,<3.0" }, + { name = "redis", marker = "extra == 'affinity-redis'", specifier = ">=5.0.1,<6" }, { name = "routellm", extras = ["serve"], marker = "extra == 'gpu'", specifier = ">=0.2.0,<1.0" }, { name = "sse-starlette", marker = "extra == 'server'", specifier = ">=3.4.1,<4.0" }, { name = "transformers", marker = "extra == 'gpu'", specifier = ">=5.8.0" }, { name = "uvicorn", extras = ["standard"], marker = "extra == 'server'", specifier = ">=0.46.0,<1.0" }, ] -provides-extras = ["server", "cli", "gpu", "tracing", "intake", "all"] +provides-extras = ["server", "cli", "gpu", "tracing", "intake", "affinity-redis", "all"] [package.metadata.requires-dev] dev = [ @@ -3708,6 +3701,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/75/1b2cfc949595e22d8c05a2aa2cfc222921f7f94177d7e8a90542f3f73b33/realtime-2.30.0-py3-none-any.whl", hash = "sha256:7c93b63d2cf99aa1da4fa8826b03b00cd32f7b38abb27ff47b19eb5dcb5707c6", size = 22376, upload-time = "2026-05-06T17:35:22.568Z" }, ] +[[package]] +name = "redis" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyjwt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/cf/128b1b6d7086200c9f387bd4be9b2572a30b90745ef078bd8b235042dc9f/redis-5.3.1.tar.gz", hash = "sha256:ca49577a531ea64039b5a36db3d6cd1a0c7a60c34124d46924a45b956e8cf14c", size = 4626200, upload-time = "2025-07-25T08:06:27.778Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/26/5c5fa0e83c3621db835cfc1f1d789b37e7fa99ed54423b5f519beb931aa7/redis-5.3.1-py3-none-any.whl", hash = "sha256:dc1909bd24669cc31b5f67a039700b16ec30571096c5f1f0d9d2324bff31af97", size = 272833, upload-time = "2025-07-25T08:06:26.317Z" }, +] + [[package]] name = "referencing" version = "0.37.0" From 1cde9572ac82f77e5e655ec8f13433ac7daf19b3 Mon Sep 17 00:00:00 2001 From: zengyuanl Date: Tue, 7 Jul 2026 19:33:50 +0000 Subject: [PATCH 06/11] docs(latency): add latency-router operations docs and Prometheus examples --- docs/internal/latency_service_routing.md | 567 ++++++++++++++++++++++ docs/internal/metrics_reference.md | 217 +++++++++ examples/prometheus/README.md | 81 ++++ examples/prometheus/prometheus.yml | 35 ++ examples/prometheus/switchyard.rules.yaml | 130 +++++ 5 files changed, 1030 insertions(+) create mode 100644 docs/internal/latency_service_routing.md create mode 100644 docs/internal/metrics_reference.md create mode 100644 examples/prometheus/README.md create mode 100644 examples/prometheus/prometheus.yml create mode 100644 examples/prometheus/switchyard.rules.yaml diff --git a/docs/internal/latency_service_routing.md b/docs/internal/latency_service_routing.md new file mode 100644 index 00000000..d6b460e0 --- /dev/null +++ b/docs/internal/latency_service_routing.md @@ -0,0 +1,567 @@ +# Latency Service Routing + +The **latency-aware router** (`type: latency_service`) routes each request to one of +several equivalent endpoints, biasing traffic toward whichever endpoint a central +**Latency Service** most recently observed as healthy and fast. It is the production +routing policy for OpenRouter-backed deployments, where a separate Latency Service +owns heartbeat probing and statistical latency profiling, and Switchyard consumes its +verdicts to make per-request steering decisions. + +This document explains how the router works, the design choices behind it, and how to +configure and operate it. + +## When to use it + +Use `latency_service` when you have **multiple interchangeable endpoints** serving the +same logical model (e.g. the same model behind different providers or regions) and you +want Switchyard to: + +- bias traffic toward the lowest-latency healthy endpoint, +- automatically steer away from degraded endpoints, and +- retry on a different endpoint when one fails, without the client ever seeing it. + +If you only have a single backend, use `passthrough`. If you want a fixed weighted +strong/weak split for benchmarking, use +[Random Routing](../routing_algorithms/random_routing.md). + +## Architecture + +The router is implemented as a single custom `LLMBackend`, +`LatencyServiceLLMBackend` (`switchyard/lib/backends/latency_service_llm_backend.py`), +rather than a monolithic routing strategy. It slots into the standard four-role chain: + +``` +[RequestProcessor*] → LatencyServiceLLMBackend → [ResponseProcessor*] → TranslationEngine +``` + +The backend owns four things: + +1. **A pool of `OpenAILLMClient` instances**, one per configured endpoint, keyed by the + endpoint's `model` ID. +2. **A thread-safe in-memory health cache** (`dict[str, EndpointHealth]` guarded by a + `threading.Lock`). +3. **A background `HealthPoller` daemon thread** + (`switchyard/lib/backends/health_poller.py`) that refreshes the cache from the + Latency Service. +4. **Endpoint-selection + retry logic** on the request hot path. + +Inbound format translation (Anthropic Messages / OpenAI Responses → OpenAI Chat) is +delegated to the Rust `TranslationEngine`, so the backend transparently accepts any +inbound wire format and only ever speaks OpenAI Chat Completions upstream. + +### Two-clock design: poll loop vs. hot path + +The defining property of this router is that **health polling and request serving run on +separate clocks**: + +- The **`HealthPoller` daemon thread** is the only thing that ever talks to the Latency + Service. It polls on a fixed interval and *writes* verdicts into the cache. +- The **request hot path** only ever *reads* the cache (under the lock). It never makes a + network call to the Latency Service. + +Health awareness adds **zero per-request latency**. Selection is a dictionary +read and a weighted random pick. The poller is a plain daemon thread (not an asyncio +task), so its lifecycle is trivial. It starts when the backend is constructed and dies +with the process. Its synchronous `httpx.Client` never contends with the event loop +serving requests. + +## The health poller + +`HealthPoller` polls `{latency_service_url}/v1/endpoints/health` every `poll_interval_s` +seconds (default 10s), passing the configured endpoint IDs as `endpoint_ids` query +params. The Latency Service responds with a verdict and a latency sample per endpoint: + +```json +{ + "endpoint_health": { + "openai/gpt-5.5": {"status": "healthy", "last_latency_ms": 420.0}, + "azure_openai/gpt-5.5": {"status": "degraded", "last_latency_ms": 980.0} + } +} +``` + +Each cache entry is an `EndpointHealth` snapshot carrying the discrete `status` and the +most recent `last_latency_ms`. The status enum is the **shared contract** between +Switchyard and the Latency Service: both sides must agree on these three string values: + +| Status | Meaning | +|---|---| +| `healthy` | Endpoint is up and within latency expectations. | +| `degraded` | Endpoint is reachable but slow or partially failing. | +| `unknown` | No current verdict (warming up, or fallback after a poll failure). | + +### Failure handling: fall back to UNKNOWN, not stale data + +If a poll fails for any reason (DNS, network, timeout, non-2xx, or a schema mismatch on +the response), the poller **resets every endpoint in the cache to `UNKNOWN`** rather than +leaving stale verdicts in place. With every endpoint at `UNKNOWN`, the router degrades to +uniform random routing across the full pool. This is the safe default when health +information is unavailable. Acting on stale "healthy" data after the Latency Service has +gone dark would be strictly worse. + +`last_latency_ms` is `None` before the first successful poll and whenever the Latency +Service reports it as null. + +## Endpoint selection + +On each request, `_select_endpoint()` picks a `model` ID from the cache in two steps: + +### 1. Tier preference: `HEALTHY > UNKNOWN > DEGRADED` + +Endpoints are bucketed by status, and the router picks from the **best non-empty tier**. +`UNKNOWN` is deliberately preferred over `DEGRADED`: an endpoint we have no verdict for is +a better bet than one the Latency Service explicitly flagged as degraded. + +### 2. Within a tier: inverse-latency weighted random + +Within the chosen tier, `_pick_by_latency()` selects an endpoint with probability +proportional to `1 / last_latency_ms`, so an endpoint the Latency Service recently saw +at 200 ms receives roughly twice the traffic of one at 400 ms. This is **weighted random, +not always-pick-the-fastest**, which spreads load and avoids stampeding a single endpoint. + +The picker falls back to **uniform random** within the tier when: + +- there is only one candidate, or +- any candidate's `last_latency_ms` is unknown (`None`), or +- any sample is non-positive (defensive: the Latency Service should never report ≤ 0). + +This keeps behavior predictable while the poller is warming up or when the upstream +reports nulls. + +### 3. Optional: session affinity (sticky routing) + +By default the router picks per **turn**. Every request runs the tier + latency steps +above independently. For a multi-turn conversation this interleaves endpoints, which lets +each endpoint's upstream prompt/KV cache lapse and forces expensive cache *re-writes* on +the next turn that lands there. + +Set `session_affinity: true` to route per **conversation** instead. The latency-aware +picker decides only the **first** turn, then pins that conversation to the endpoint +that served it. Every later turn reuses that endpoint (the affinity fast-path bypasses +tier + latency selection entirely), so the upstream cache stays warm for the life of +the conversation. + +- **Conversation identity.** A streaming proxy holds no session ID. The client re-sends + the full history each turn. `session_key_from_body()` + (`switchyard/lib/session_key.py`) derives a stable key by hashing the stable harness + prefix made from the system prompt + the first user message. Later turns only append, so + every turn of one conversation hashes the same; distinct conversations differ. +- **Health overrides the pin.** A pin is reused only while its endpoint is `HEALTHY` or + `UNKNOWN`. If it goes `DEGRADED` (or the call fails this request), the next turn + re-routes through normal selection and re-pins to whatever serves it. Locality yields + to health, so a session is never funneled into a failing endpoint. +- **Bounded memory.** Pins live in an in-process LRU map (the L1) capped at + `affinity_max_sessions` (default `10_000`); the least-recently-used pin is evicted past + the cap. +- **Cross-worker stickiness (optional shared store).** The L1 map is per process, so a + multi-worker/multi-pod deployment may pin the same conversation differently per worker, + and pins are lost on pod churn. Set `affinity_store: "redis"` (with `affinity_store_url`) + to add a shared, persistent L2: on an L1 miss the pin is read through from the store and + warmed back into L1, and every pin is written through to it. Pins then survive across + replicas and pod restarts (bounded by `affinity_store_ttl_seconds`, refreshed on each + write). The L2 is **best-effort** — an error or timeout falls back to L1 / unpinned and + never fails a request; fail-open operations are counted on + `switchyard_affinity_l2_errors_total`, and a circuit breaker (3 consecutive failures → + skip the store for 10 s, then probe; `switchyard_affinity_l2_breaker_open` gauge) keeps + a store outage from taxing every request with timeout waits. The store connection is released by the backend's + `shutdown()` (the server lifespan teardown hook awaits it). Requires the + `switchyard[affinity-redis]` extra. (Alternatively, front the workers with a + session-affinity load balancer.) + +The `switchyard.route_decision` span tags each pick with `switchyard.affinity_hit` so you +can measure the warm-reuse rate. Affinity is **off by default**; existing per-turn latency +routing is unchanged unless opted in. + +#### Deploying the shared store: the operational contract + +What an operator needs to know to provision and run the Redis L2. Everything below is +verified against the implementation (`RedisPinStore`, `SessionAffinity`). + +**Topology.** A **standalone Redis endpoint only** (the client is +`redis.asyncio.from_url`): a single instance, a primary endpoint of a replicated setup, +or a managed single-endpoint tier. **Redis Cluster and Sentinel URLs are not +supported** — cluster `MOVED` redirects surface as fail-open errors, meaning requests +keep succeeding while stickiness silently degrades. A sustained +`switchyard_affinity_l2_errors_total` rate from the first request is the +mis-provisioning signal; watch it when first enabling the store. + +**Auth and TLS.** URL-driven: `rediss://` for TLS, credentials inside the URL. Route +YAML expands `${ENV}` references in every string value, so keep the URL in a secret and +configure `affinity_store_url: "${AFFINITY_REDIS_URL}"` rather than an inline literal. + +**What is stored.** Key = `affinity_key_prefix` + a 16-hex-char hash of the +conversation's stable prefix (system prompt + first user message); value = the pinned +endpoint id string. **No prompt text, message content, or user identifier leaves the +process** — a pin is ~50–100 bytes. + +**Traffic and sizing.** One `GET` per request whose conversation isn't in the local L1 +(first turn on a pod), and one `SET` (which refreshes the TTL) per successful request. +Working-set memory ≈ active conversations × ~150 B, bounded by the sliding +`affinity_store_ttl_seconds` (default 1 h). **Persistence (RDB/AOF) and HA are optional**: +losing the store only resets stickiness — conversations re-route cold and re-pin — it +never affects correctness. `maxmemory-policy allkeys-lru` is the recommended eviction +policy; an evicted pin costs one cold re-route. + +**Failure envelope.** Every store operation is bounded by a 0.1 s socket/connect +timeout (a colocated Redis answers in well under 1 ms, so this is ~100× headroom) and +fails open — requests never fail because of the store. A **circuit breaker** caps the +outage tax: after 3 consecutive failures, L2 operations are skipped without a network +attempt for a 10 s cooldown, then one operation probes; success closes the breaker, +failure re-arms it. Worst case per pod during an outage is therefore a brief window of +up to ~0.2 s added per request (one `GET` on the L1 miss + one `SET` after success) +before the breaker opens, then ~zero added latency with one 0.1 s probe per 10 s — +stickiness degrades to per-pod L1 throughout. Alert on +`switchyard_affinity_l2_breaker_open == 1` (the outage signal) and on +`rate(switchyard_affinity_l2_errors_total[5m])` (degradation without a full trip — +note the rate drops to ~0.1/s per pod while the breaker is open, since only probes +fail). Pod readiness (`/health`, `is_ready()`) never depends on Redis. + +**Isolation.** Deployments or routes sharing one Redis should set distinct +`affinity_key_prefix` values (or separate logical databases). A pin is honored whenever +its stored endpoint id exists in the reading route's endpoint set, so shared prefixes +cross-pollinate stickiness between routes that reuse endpoint ids. + +**Config changes, upgrades, rollback.** A persisted pin whose endpoint id is no longer +in the route's endpoint list is ignored gracefully (the next turn re-routes and +re-pins) — shrinking or renaming the endpoint set needs no store cleanup. The +conversation-key derivation is not a cross-version contract: a Switchyard upgrade may +invalidate existing pins (one-time cold re-route, self-healing). To roll back, set +`affinity_store: "memory"`; stale keys expire via TTL, and `FLUSHDB` is always safe +(stickiness reset only). + +The deterministic LLM-classifier route also supports `affinity_warmup_turns` to delay +when a tier becomes sticky. Latency-service routing does **not** use that knob. Its +affinity objective is endpoint cache reuse, so it pins after the first successful +endpoint selection. + +## Retry and failover + +A single client request may try up to `1 + max_retries` endpoints (default `max_retries` +is 2, so 3 attempts). On each attempt: + +1. Select an endpoint. If selection returns one already tried this request (and untried + endpoints remain), pick uniformly at random from the **untried** set. Dedup prevents + wasting an attempt re-hitting an endpoint that just failed. +2. Stamp the endpoint's `upstream_model` into `body["model"]` and call the upstream. +3. On a **transient** failure such as `429` (rate limit), `408` (request timeout), any + `5xx`, or a network / pre-status / SDK error, log it, record the error in stats, increment + the outcome metric, and continue to the next endpoint. These are exactly the faults a + health-aware router should absorb by trying a different endpoint. +4. On a **4xx client error** (`400`, `401`, `403`, `404`, `409`, `413`, `415`, `422`, …) + the loop **fails fast**: it does *not* retry. The request itself is malformed or + unauthorized, so every replica rejects the same payload identically; retrying only adds + latency before surfacing the same status. (Context-window overflow surfaces as a `400` + here and is also passed straight through. The `latency_service` route does not + participate in the chain-level evict-and-retry described in + [Context-Window Handling](../operations/context_window.md).) + +If an attempt succeeds **after** at least one failure, the router records a +`retry_recovered` outcome. This is direct evidence the steering logic rescued a request that +would otherwise have failed. + +Whenever the request ends on an upstream HTTP error, either a fail-fast 4xx or an +exhausted retryable error (e.g. a `503` after retries are exhausted), the backend records the +upstream status code and body on `ctx` so the endpoint passes the real upstream status +through to the client rather than masking it as a generic `500`. + +## Configuration + +`LatencyServiceBackendConfig` owns the router's policy, retry, and optional +session-affinity settings. In CLI YAML, configure the Python +`type: latency_service` router in a `routes:` bundle; there are no +policy-specific CLI flags. Run it with: + +```bash +switchyard --routing-profiles my_routes.yaml -- serve --port 4100 +``` + +The Rust `type: latency-service` profile loaded by `switchyard serve --config` +is a separate schema and does not yet expose `session_affinity` or +`affinity_max_sessions`. + +### Config schema + +`LatencyServiceBackendConfig` +(`switchyard/lib/config/latency_service_backend_config.py`): + +| Field | Default | Purpose | +|---|---|---| +| `latency_service_url` | `""` | Base URL of the Latency Service. | +| `endpoints` | `[]` | The endpoints to route across (see below). At least one is required. | +| `poll_interval_s` | `10.0` | How often the background poller refreshes health. | +| `poll_timeout_s` | `5.0` | Timeout for each health API call. | +| `max_retries` | `2` | Extra endpoints to try on failure (total attempts = `1 + max_retries`). | +| `credential_policy` | `"configured_endpoint"` | Credential precedence for upstream calls. The default uses each endpoint's configured `api_key`; set `"caller_override"` only for BYO-key deployments where caller headers should replace endpoint keys. | +| `session_affinity` | `false` | Pin each conversation to one endpoint after the first turn (sticky routing: keeps the upstream cache warm). See [Endpoint selection §3](#3-optional-session-affinity-sticky-routing). | +| `affinity_max_sessions` | `10_000` | LRU cap on the in-process pin map (L1) when `session_affinity` is on. | +| `affinity_store` | `"memory"` | Shared L2 pin store behind the L1. `"memory"` = per-process only; `"redis"` shares pins across workers/pods and persists them across pod churn (requires `session_affinity: true`, `affinity_store_url`, and the `affinity-redis` extra). | +| `affinity_store_url` | `None` | Connection URL for the shared store (e.g. `redis://host:6379/0`). Required when `affinity_store` is `"redis"`. | +| `affinity_store_ttl_seconds` | `3_600` | Expiry for a shared pin; refreshed on each write, so an active conversation slides its TTL. | +| `affinity_key_prefix` | `"swyd:pin:"` | Namespace prefix for shared-store keys. | +| `enable_stats` | `true` | Wire the stats processors + `/metrics` exposition. | + +Each `LatencyServiceEndpoint`: + +| Field | Required | Purpose | +|---|---|---| +| `model` | yes | **Latency Service lookup key**: must be unique across the endpoint list. Also the value stamped into `body["model"]` unless `upstream_model` is set. | +| `upstream_model` | no | Override for `body["model"]` sent upstream. Defaults to `model`. | +| `api_key` | no | API key for the backing LLM API. This is the default upstream credential. | +| `base_url` | no | Base URL for the backing LLM API (include `/v1`). | +| `timeout` | no | Per-request timeout in seconds. | + +### Why two `model` fields? + +The Latency Service registers each endpoint under an ID *it* picked (e.g. +`openrouter-primary/openai/gpt-4o`). The upstream gateway may expect the OpenRouter +model name for the same destination (e.g. `openai/gpt-4o`). So: + +- **`model`** is the Latency Service lookup key (and the metrics label). +- **`upstream_model`** is what gets stamped into `body["model"]` on the outbound call. + +When they're equal, `upstream_model` can be omitted. + +### Example + +Create an OpenRouter key at and export it as +`OPENROUTER_API_KEY` before running this example. + +```yaml +routes: + gpt-4o: + type: latency_service + latency_service_url: https://latency-service.example.com + poll_interval_s: 10.0 + poll_timeout_s: 5.0 + max_retries: 2 + endpoints: + - model: openrouter-primary/openai/gpt-4o # Latency Service endpoint_id + upstream_model: openai/gpt-4o # OpenRouter model name to call + base_url: https://openrouter.ai/api/v1 + api_key: ${OPENROUTER_API_KEY} + - model: openrouter-backup/openai/gpt-4o + upstream_model: openai/gpt-4o + base_url: https://openrouter.ai/api/v1 + api_key: ${OPENROUTER_API_KEY} +``` + +A runnable version of this config, with extensive inline notes, lives at +[`experiments/latency_routing.yaml`](../../experiments/latency_routing.yaml). +A production-style Kubernetes manifest lives at +[`examples/k8s/latency-aware-deployment.yaml`](../../examples/k8s/latency-aware-deployment.yaml). + +### BYO-key (multi-tenant) mode + +By default, caller `Authorization: Bearer ` and `x-api-key` headers do not replace +the configured endpoint credentials. This keeps server-owned latency-service deployments +from accidentally forwarding a client placeholder or unrelated proxy credential upstream. + +For a multi-tenant BYO-key deployment, set `credential_policy: caller_override` on the +latency-service route. In that mode, a non-empty caller key is forwarded as the upstream +credential for that request, and the endpoint `api_key` is used only when the caller omits +the header. Credential precedence is **`Authorization` > `x-api-key` > endpoint +`api_key`**. + +## Response fidelity (Responses API) + +For an endpoint configured `request_type: openai_responses`, the backend calls the +upstream `/v1/responses` surface and returns the upstream payload **verbatim** in both +modes: + +- **Non-streaming** calls return the upstream's exact JSON body — the raw HTTP response + is used instead of round-tripping through the OpenAI SDK's typed model. +- **Streaming** calls forward the upstream's SSE frames as **verbatim strings** + (`RawSSEFrameStream` in `switchyard/lib/llm_client.py`): no typed-event parse happens, + so per-event provider extras, explicit nulls, event names, and comment keep-alives all + survive. Frames are byte-equivalent modulo CRLF → LF line normalization. Usage + accounting still works — the stats layers parse the frame's `data:` payload to find + `response.completed` usage without altering what flows to the client. + +When the inbound request is also Responses-format, the terminal translation +short-circuits (same format in and out), so the client receives the upstream payload +as-is. **Cross-format** calls (e.g. a chat client served by a Responses endpoint, or +vice versa) are *re-synthesized* by the translation engine and are inherently lossy — +fields the upstream never produced in the target format cannot be invented. For +full-fidelity Responses routing, configure every endpoint on the route with +`request_type: openai_responses`. + +### Field-fidelity matrix (Responses API) + +How each field class fares per path. "Exact" means byte-equivalent to the upstream +(streaming: modulo CRLF normalization); "synthesized" means the translation engine +rebuilds the payload from its neutral representation and only mapped fields survive. + +| Field class (examples) | Same-format, non-streaming | Same-format, streaming | Cross-format (either direction) | +|---|---|---|---| +| Standard scalars (`id`, `model`, `status`, `created_at`) | exact | exact | synthesized (`id`/`model` mapped; `created_at` → chat `created` defaults to `0`) | +| Sampling/config echoes (`temperature`, `top_p`, `store`, `text`) | exact | exact | dropped (no chat-format equivalent) | +| `reasoning` config + explicit-`null` fields | exact (nulls preserved) | exact (nulls preserved) | dropped | +| `usage` (incl. `*_tokens_details`) | exact | exact | token counts + reasoning detail mapped; cached-token detail drops | +| Output content (`output[]`, text deltas) | exact | exact | mapped (text/tool calls translate; unmappable block types drop) | +| Provider extras (unknown keys, e.g. Azure `content_filter_results`) | exact | exact | dropped | +| SSE event names / keep-alive comments | n/a | exact | re-framed to the target format's event contract | +| `metadata`, `previous_response_id` | exact | exact | dropped | + +Wire-level proofs live in +`tests/test_upstream_error_passthrough.py` +(`test_responses_body_returned_exactly_on_the_wire`, +`test_responses_stream_forwarded_verbatim_on_the_wire`, +`test_azure_flavored_responses_stream_preserved_on_the_wire`) against +OpenAI-shaped and Azure-shaped samples; validation against live Inference Hub +captures remains an IH-side step. + +## Failure-source annotation + +When a request fails, the error response carries two headers so clients and +observability can tell **which layer** the failure came from without parsing bodies: + +| Header | Values | Meaning | +|---|---|---| +| `x-switchyard-error-source` | `switchyard` \| `provider` | `provider`: an upstream LLM failure passed through (HTTP error, or the 500 rendered for a status-less network fault). `switchyard`: this proxy rejected or failed the request itself — credential-policy 401 (`missing_caller_api_key`), translation 400 (`invalid_value`), model-not-found 404, context-window 400, body validation 400, unexpected internal 500. | +| `x-switchyard-upstream-model` | model id | The model actually attempted upstream when the failure happened. Present only when a routing selection took place. | + +The provider error **body** still passes through verbatim — the annotation is +header-only, deliberately, so the transparent-proxy contract on bodies is preserved. +The same vocabulary appears as `switchyard.error_source` / `switchyard.upstream_model` +tags on failed `switchyard.upstream_attempt` spans and as `error_source` / +`upstream_model` fields on the per-event error log (below). + +Layering note: Switchyard can only distinguish *itself* from *its upstreams*. A proxy in +front of Switchyard (e.g. LiteLLM) should tag its own failures the same way and +propagate these headers from below. Mid-stream failures after HTTP 200 has been +committed cannot carry headers and are not annotated today. + +## Observability + +When `enable_stats` is `true` (the default), `LatencyServiceProfileConfig` wires a +`StatsRequestProcessor` + `StatsResponseProcessor` pair sharing one `StatsAccumulator`. +Because `LatencyServiceLLMBackend` is a Python-only backend, profile assembly cannot wrap +it with the Rust-native `StatsLlmBackend`; instead it records success / error / +call-latency in-place into the same accumulator the response processor records token usage +into. It also sets `ctx.selected_model` and `ctx.backend_call_latency_ms` so per-endpoint +attribution and routing-overhead figures land correctly on `/metrics`. + +The backend publishes its in-memory health cache and poll-loop health to `/metrics`: + +| Metric | Type | Meaning | +|---|---|---| +| `switchyard_endpoint_status{model,status}` | gauge | Current verdict per endpoint; exactly one status row per model is `1`. | +| `switchyard_endpoint_last_latency_ms{model}` | gauge | Last latency sample per endpoint. Absent when the upstream reported null. | +| `switchyard_latency_service_poll_ok` | gauge | `1` iff the most recent poll succeeded. | +| `switchyard_latency_service_poll_age_seconds` | gauge | Seconds since the last successful poll. Absent before the first success. | +| `switchyard_latency_service_polls_total` | counter | Total successful polls. | +| `switchyard_latency_service_poll_failures_total` | counter | Total failed polls; each resets the cache to `unknown`. | +| `switchyard_affinity_hits_total` | counter | Turns served by an existing session-affinity pin (warm reuse). **Emitted only when `session_affinity` is on.** | +| `switchyard_affinity_misses_total` | counter | First/unpinnable turns routed by the latency-aware picker while affinity was on. Reuse rate = `hits / (hits + misses)`. **Emitted only when `session_affinity` is on.** | +| `switchyard_affinity_l2_hits_total` | counter | Pins resolved from the shared (L2) store after an in-process miss — cross-worker/churn warm reuse. **Emitted only when a shared affinity store is configured.** | +| `switchyard_affinity_l2_errors_total` | counter | Shared-store get/put operations that failed open; routing fell back to in-process pins. Alert on a sustained rate — the store is degraded while requests keep succeeding. **Emitted only when a shared affinity store is configured.** | + +See the [Metrics Reference](metrics_reference.md) for full label semantics, PromQL recipes +(traffic share per endpoint, retry-rescue rate), and a triage cheatsheet. + +### Expected routing overhead + +`switchyard_routing_overhead_ms{quantile}` is the source of truth for what Switchyard +itself adds to a request: per request it records `total_latency − backend_call_latency`, +which bundles format translation, endpoint selection, body mutation, response wrapping, +and retry wall time — everything except the upstream call. + +Reference figures, measured on an idle dev box over loopback (stub upstream + stub +Latency Service, non-streaming `/v1/chat/completions`, sequential; commit `0289a415`): + +| Figure | Value | +|---|---| +| `switchyard_routing_overhead_ms{quantile="0.5"}` | ~0.3 ms | +| `switchyard_routing_overhead_ms{quantile="0.99"}` | ~7 ms | +| Mean overhead (`_sum / _count`) | ~2 ms | +| Client-observed added wall time vs calling the upstream directly | ~2 ms mean | + +Expected order of magnitude is therefore **single-digit milliseconds**. When an +end-to-end comparison ("direct provider call" vs "through the stack") shows a much +larger delta — e.g. ~100 ms — the difference lives **outside** this metric: + +* the extra network hop and TLS handshake from the client to the Switchyard pod, +* any fronting layer (LiteLLM / inference-gateway) in front of Switchyard, +* per-pod queueing under concurrency — a single event loop serializes CPU work, so + client wall time grows with in-flight load while per-request overhead does not + (at loopback concurrency 10 the same box shows ~22 ms client p50 with an unchanged + overhead summary), +* upstream time-to-first-token variance between the two measurement runs. + +Attribute before optimizing: read the deployment's own summary first. + +```promql +# Median / p99 Switchyard-internal overhead +switchyard_routing_overhead_ms{quantile="0.5"} +switchyard_routing_overhead_ms{quantile="0.99"} + +# Mean over a window +rate(switchyard_routing_overhead_ms_sum[5m]) / rate(switchyard_routing_overhead_ms_count[5m]) +``` + +Two caveats. For streaming responses `switchyard_total_latency_ms` (and hence the +overhead summary) measures the **full turn**, not time-to-first-token — TTFT is what +interactive users feel, so don't read a long generation as router overhead. And the +summary is per-process: in a multi-pod deployment each pod reports its own. + +The repeatable harness is the CI perf benchmark (`.github/workflows/perf.yml`): aiperf +against a no-op route measures the full server path with zero backend cost, non-streaming +and streaming, at configurable concurrency. + +### Per-event error log (Loki) + +`/metrics` answers *how many* of each error code, but Prometheus stores aggregates. It +carries no per-event timestamps. For audit, replay, or a "what failed and when" panel, +each failed upstream attempt also emits one structured JSON line on the +`switchyard.upstream_errors` logger +(`switchyard/lib/endpoints/upstream_error_log.py`): + +```json +{"event":"upstream_attempt_failed","timestamp":"2026-05-29T12:00:00+00:00","model":"openai/gpt-5.5","upstream_model":"gpt-5.5","attempt":1,"status_code":429,"code":"429","outcome":"retryable_error","error_source":"provider","error_type":"APIStatusError","error":"..."} +``` + +`code` and `outcome` mirror the labels on `switchyard_upstream_attempts_total`, so the log +joins cleanly to the counter; `status_code` is `null` (and `code="none"`) for a non-HTTP +failure. Because the deployment uses plain-text logging, the message *is* the JSON object. +A Loki query parses it with `| json` and zero formatter config: + +```logql +{app="switchyard"} | json | event="upstream_attempt_failed" +``` + +`is_ready()` returns `True` once the poller has completed at least one successful poll, +which makes it useful as a readiness gate during startup. + +## Testing the router + +Three tiers, cheapest first. The `switchyard-latency-router-e2e` skill carries the +runnable commands and helper scripts for each. + +1. **Offline unit suite** (no keys): `tests/test_latency_service_*.py`, + `tests/test_session_key.py`, `tests/test_outcome_metrics.py`, and + `tests/test_upstream_error_passthrough.py`: full coverage of selection, tiering, + inverse-latency weighting, retry/fail-fast, session affinity, the health poller, + metrics, and spans. +2. **In-repo integration** (`tests/e2e/test_latency_service_llm_backend.py`, marked + `integration`): an in-process mock Latency Service feeds verdicts while the LLM + calls hit OpenRouter. Requires `OPENROUTER_API_KEY` (skips without it). +3. **Real-world**: point a real backend at a mock or live Latency Service and confirm + it routes. A mock LS pointed at `openrouter.ai` proves the + poll → tier → weighting path with no API key — a keyless dispatch returns `401`, + which confirms traffic reached the host. + +## Source map + +| File | Responsibility | +|---|---| +| `switchyard/lib/backends/latency_service_llm_backend.py` | Selection, session-affinity pins, retry, stats/metrics emission, per-attempt error log. | +| `switchyard/lib/session_key.py` | `session_key_from_body()`: stable per-conversation key for session affinity. | +| `switchyard/lib/session_affinity.py` | `SessionAffinity`: L1 (in-process LRU) + optional best-effort L2 pin store. | +| `switchyard/lib/affinity_pin_store.py` | `AffinityPinStore`: async protocol for a shared/persistent L2 store. | +| `switchyard/lib/redis_pin_store.py` | `RedisPinStore`: Redis-backed shared L2 (lazy `redis`, `affinity-redis` extra). | +| `switchyard/lib/endpoints/upstream_error_log.py` | Structured per-attempt failure log (`switchyard.upstream_errors`, Loki). | +| `switchyard/lib/backends/health_poller.py` | `EndpointHealthStatus`, `EndpointHealth`, `HealthPoller` daemon. | +| `switchyard/lib/config/latency_service_backend_config.py` | `LatencyServiceEndpoint`, `LatencyServiceBackendConfig`. | +| `switchyard/lib/profiles/latency_service.py` | `LatencyServiceProfileConfig`: profile assembly + stats wiring. | +| `tests/test_latency_service_llm_backend.py` | Selection, retry, tier-preference tests. | +| `tests/test_latency_service_health_metrics.py` | Poller and metrics tests. | diff --git a/docs/internal/metrics_reference.md b/docs/internal/metrics_reference.md new file mode 100644 index 00000000..f2281b5d --- /dev/null +++ b/docs/internal/metrics_reference.md @@ -0,0 +1,217 @@ +# Switchyard Metrics Reference + +Operational reference for the Prometheus exposition served by a Switchyard +deployment. Pair with [`examples/prometheus/`](../../examples/prometheus/) for +a drop-in scrape config and starter alert rules. + +## Endpoint + +| Property | Value | +|---|---| +| Path | `GET /metrics` (HTTP path is `/metrics`, **not** `/v1/metrics`) | +| Content-Type | `text/plain; version=0.0.4; charset=utf-8` | +| Format | Prometheus text format 0.0.4 | +| Auth | None. Designed as a public scrape endpoint | +| Default scrape interval | 15s (the Latency Service poll cycle is 10s by default, so finer scrape resolution captures no extra state) | + +!!! note "Availability: legacy route-bundle serve only" + `GET /metrics` is served by the **legacy route-bundle path** + (`switchyard --routing-profiles PATH serve`). The recommended v2 profile server + (`switchyard serve --config`) returns **404** for `/metrics` — use `GET /v1/stats` + (or its alias `GET /v1/routing/stats`) on that path instead. + +A JSON variant of the same underlying data lives at `GET /v1/stats`, with +`GET /v1/routing/stats` as a backwards-compatible alias. + +## Top-line gauges (no labels) + +| Metric | Type | Meaning | +|---|---|---| +| `switchyard_total_requests` | gauge | Total upstream call attempts recorded since process start. **Per-attempt**, not per-client-request. One client request that retries-then-succeeds increments by 2. | +| `switchyard_total_errors` | gauge | Total upstream call attempts that errored, including those absorbed by retry. | + +## Per-endpoint counters + +The `model` label is the latency-service endpoint id (`openai/gpt-5.5`, +`azure_openai/gpt-5.5`, etc.). + +The `tier` label is optional: present only when a routing factory +supplied one. The `random_routing` factory emits `tier="strong"|"weak"`; +the `latency_service` factory does not. + +| Metric | Type | Meaning | +|---|---|---| +| `switchyard_requests_total{model}` | counter | Successful upstream call attempts per endpoint. | +| `switchyard_errors_total{model}` | counter | Failed upstream call attempts per endpoint (any cause). | +| `switchyard_prompt_tokens_total{model}` | counter | Prompt-token billing per endpoint. | +| `switchyard_completion_tokens_total{model}` | counter | Completion-token usage per endpoint. | +| `switchyard_cached_tokens_total{model}` | counter | Cached prompt tokens per endpoint. | +| `switchyard_cache_creation_tokens_total{model}` | counter | Cache-creation tokens per endpoint. | +| `switchyard_reasoning_tokens_total{model}` | counter | Reasoning tokens per endpoint. | + +## Per-endpoint latency summaries + +Each summary emits `{quantile="0.5"}` and `{quantile="0.99"}` rows plus +`_sum` and `_count`. + +| Metric | Type | Meaning | +|---|---|---| +| `switchyard_model_call_latency_ms{model,quantile}` | summary | Upstream LLM call duration. | +| `switchyard_total_latency_ms{model,quantile}` | summary | End-to-end request latency (request entry → response complete). For streaming responses this is full-turn time, **not** time-to-first-token. | + +## Routing overhead (global, no model label) + +| Metric | Type | Meaning | +|---|---|---| +| `switchyard_routing_overhead_ms{quantile}` | summary | `total_latency − backend_latency`, across all calls. Bundles format translation, endpoint selection, body mutation, response wrapping, and retry wall time. | + +Healthy traffic typically sits at p50 ≈ 0.4 ms, p99 ≈ 0.6 ms. + +## Latency Service state (gauges: latency-service chains only) + +Published from the in-memory health cache the `LatencyServiceLLMBackend` +maintains, refreshed on each successful poll of the Latency Service. + +| Metric | Type | Meaning | +|---|---|---| +| `switchyard_endpoint_status{model,status}` | gauge | Current Latency Service verdict per endpoint. `status` is one of `healthy`, `degraded`, `unknown`. Exactly one row per `model` is `1`; the rest are `0`, so `sum by (status)` gives a clean count of endpoints in each state. | +| `switchyard_endpoint_last_latency_ms{model}` | gauge | Last latency sample the Latency Service reported for this endpoint. **Absent** when the upstream reported `null`, and absence is meaningful. | + +## Latency Service poll health (no labels: latency-service chains only) + +| Metric | Type | Meaning | +|---|---|---| +| `switchyard_latency_service_poll_ok` | gauge | `1` iff the most recent poll succeeded. `0` means the next routing decisions are based on the cache-reset-to-unknown fallback state. | +| `switchyard_latency_service_poll_age_seconds` | gauge | Monotonic seconds since the last successful poll. **Absent** before the first success; combined with `poll_ok=0`, this distinguishes "never polled" from "polled but recently failed". | +| `switchyard_latency_service_polls_total` | counter | Total successful polls since process start. | +| `switchyard_latency_service_poll_failures_total` | counter | Total failed poll attempts. Each failure resets every endpoint in the cache to `unknown`. | + +## Session affinity (no labels: latency-service chains with `session_affinity` on) + +These counters are **absent unless `session_affinity: true`**, so the metric surface stays clean for the default per-turn-routing case. + +| Metric | Type | Meaning | +|---|---|---| +| `switchyard_affinity_hits_total` | counter | Conversation turns served by an existing session-affinity pin, meaning the upstream prompt/KV cache was reused. Counted once per request (first attempt only), so failover retries don't inflate it. | +| `switchyard_affinity_misses_total` | counter | First turns of a conversation, or turns whose pin was broken by a health verdict, routed by the latency-aware picker. | +| `switchyard_affinity_l2_hits_total` | counter | Pins resolved from the shared (L2) affinity store after an in-process (L1) miss — cross-worker/churn warm reuse. **Also requires a shared store** (`affinity_store: "redis"`). | +| `switchyard_affinity_l2_errors_total` | counter | Shared-store get/put operations that failed open; routing fell back to in-process pins. A sustained rate means the store is degraded while requests keep succeeding — this is the alerting signal. While the breaker is open, only recovery probes fail, so the rate drops to ~1 per cooldown per pod. **Also requires a shared store.** | +| `switchyard_affinity_l2_breaker_open` | gauge | `1` while the shared-store circuit breaker is open (operations skipped without a network attempt after 3 consecutive failures; one probe per 10 s cooldown), `0` when closed. The unambiguous store-outage signal. **Also requires a shared store.** | + +Warm-reuse rate (the fraction of turns that hit a warm endpoint): + +```promql +rate(switchyard_affinity_hits_total[5m]) + / (rate(switchyard_affinity_hits_total[5m]) + rate(switchyard_affinity_misses_total[5m])) +``` + +## Outcome counters for error-rate ratios + +The `outcome` label takes exactly three values: + +* `success` = HTTP 2xx +* `retryable_error` = HTTP 429 / 500 / 504 +* `other_error` = everything else (400, 401, 403, 422, …) + +| Metric | Type | Meaning | +|---|---|---| +| `switchyard_client_responses_total{outcome}` | counter | HTTP responses returned to clients on the LLM-serving routes (`/v1/chat/completions`, `/v1/messages`, `/v1/responses`). The denominator for the **router-served** error rate. | +| `switchyard_upstream_attempts_total{outcome,code}` | counter | Individual upstream call attempts. One client request can produce N attempts via retry. The denominator for the **direct-to-endpoint** baseline error rate. The `code` label carries the raw upstream HTTP status for plotting the error-code distribution (see below). | +| `switchyard_router_retry_recovered_total` | counter | Requests whose first upstream attempt failed but a subsequent attempt succeeded. This is direct evidence the routing logic rescued the request. | +| `switchyard_latency_upstream_attempts_total{requested_model,upstream_model,outcome,code}` | counter | Latency-service chains only: the per-model complement of `switchyard_upstream_attempts_total` (which stays model-free). `requested_model` is the client-supplied model bounded to a config-derived id — the route id (`route_model`, e.g. `nvidia/switchyard/gpt-5.4`) or a configured endpoint id — with `other` as the fallback sentinel. `upstream_model` is the selected endpoint's upstream name. Answers "for route X, which upstream failed or succeeded". | + +### The `code` label on `switchyard_upstream_attempts_total` + +`code` is the raw upstream HTTP status as a string: `"200"`, `"429"`, +`"500"`, `"504"`, etc. Two special values: + +* `code="none"`: a non-HTTP failure (network error, connection reset, + pre-status timeout). The attempt never received a status line, so there + is no code. These also count as `outcome="retryable_error"`. +* `code="4xx"` / `code="5xx"` / `code="1xx"` / `code="3xx"` / `code="other"`: + an HTTP code outside the known-codes allowlist, clamped to its class so + a misbehaving upstream cannot blow up label cardinality. + +`outcome` is fully determined by `code`, so adding the label does not +multiply series. You get one series per distinct code either way. The +canonical codes (`200`, `429`, `500`, `504`, `none`) are seeded at `0` so +their time series exist from process start (a `rate()` over a never-seen +counter reads as "no data", not zero). + +## Computing the success-criterion ratios + +```promql +# Router error rate (the rate clients see) +router_error_rate = + sum(rate(switchyard_client_responses_total{outcome="retryable_error"}[5m])) + / sum(rate(switchyard_client_responses_total[5m])) + +# Direct-endpoint error rate (what clients would have seen without the router) +direct_error_rate = + sum(rate(switchyard_upstream_attempts_total{outcome="retryable_error"}[5m])) + / sum(rate(switchyard_upstream_attempts_total[5m])) + +# Headline metric: positive value means the router is reducing client errors +error_rate_reduction = direct_error_rate − router_error_rate + +# Live Endpoint Routing rescue rate +rate(switchyard_router_retry_recovered_total[5m]) + +# Traffic share per endpoint (sanity-check inverse-latency weighting) +sum by (model) (rate(switchyard_requests_total[5m])) + / ignoring(model) group_left sum(rate(switchyard_requests_total[5m])) + +# Error-code distribution over time (stack the series in a Grafana time-series panel) +sum by (code) (rate(switchyard_upstream_attempts_total{code!="200"}[5m])) + +# Same, as a 100%-stacked share rather than absolute rates +sum by (code) (rate(switchyard_upstream_attempts_total{code!="200"}[5m])) + / ignoring(code) group_left +sum (rate(switchyard_upstream_attempts_total{code!="200"}[5m])) + +# Per-upstream outcome breakdown for one route — "for nvidia/switchyard/gpt-5.4, +# which upstream provider failed or succeeded?" (latency-service chains) +sum by (upstream_model, outcome, code) ( + rate(switchyard_latency_upstream_attempts_total{requested_model="nvidia/switchyard/gpt-5.4"}[5m]) +) +``` + +> **Note:** because `switchyard_upstream_attempts_total` now carries the +> `code` label, always wrap a bare selector in `sum()` (as the ratio +> queries above do) when you want a layer total. Otherwise the selector +> returns one series per code. + +The ready-to-deploy alert rules implementing these expressions live in +[`examples/prometheus/switchyard.rules.yaml`](../../examples/prometheus/switchyard.rules.yaml). + +## Cardinality + +All labels are bounded enums. No per-request or per-user values escape +into label space. + +| Label | Values | Where | +|---|---|---| +| `model` | One per configured endpoint, typically 2–6 per deployment. | All per-endpoint metrics. | +| `status` | Exactly 3: `healthy`, `degraded`, `unknown`. | `switchyard_endpoint_status` | +| `outcome` | Exactly 3: `success`, `retryable_error`, `other_error`. | Outcome counters | +| `code` | Bounded: the known-code allowlist (`200`, `400`, `401`, `403`, `404`, `408`, `409`, `422`, `429`, `500`, `502`, `503`, `504`), plus `none` and the per-class buckets `1xx`/`2xx`/`3xx`/`4xx`/`5xx`/`other`. About 20 values max. | `switchyard_upstream_attempts_total` | +| `requested_model` | Bounded to config-derived ids: the route id (`route_model`) plus configured endpoint ids, else the `other` sentinel. | `switchyard_latency_upstream_attempts_total` | +| `upstream_model` | One per configured endpoint (the endpoint's upstream name). | `switchyard_latency_upstream_attempts_total` | +| `quantile` | Exactly 2: `0.5`, `0.99`. | All summaries | +| `tier` | Small enumerated set, optional. Not emitted by latency-service chains. | Per-endpoint counters/summaries on routing chains that supply it | + +For a latency-service deployment with `N` endpoints, expect roughly +`11N + 17` series at startup (five `code` series are seeded), growing by at +most a dozen more as additional upstream status codes appear. That is about 39 +series for two endpoints. Well within single-Prometheus capacity. + +## Triage cheatsheet + +| Symptom on `/metrics` | Likely cause | +|---|---| +| `model=""` rows appear | The per-endpoint attribution wiring regressed. `ctx.selected_model` is not being set by the backend. | +| All counters at 0 after warm-up | Server just started with no traffic, or the scraper is hitting the wrong port. | +| `switchyard_latency_service_poll_ok` stuck at `0` | DNS / network unreachability to the Latency Service, **or** a schema mismatch on the LS response. Check server logs for `"Health poller: failed to reach Latency Service"`. | +| `switchyard_endpoint_last_latency_ms{...}` absent for an endpoint | The Latency Service reported `last_latency_ms: null`, or the endpoint was not returned in the most recent poll response. | +| `switchyard_routing_overhead_ms_count` stuck at `0` | The backend is not publishing `ctx.backend_call_latency_ms` (regression of the Python-backend routing-overhead wiring). | +| `switchyard_client_responses_total{outcome="retryable_error"}` rising | Either the upstream is genuinely flaky (cross-check `switchyard_endpoint_status`), or retries are exhausting (compare with `switchyard_router_retry_recovered_total` rate). | diff --git a/examples/prometheus/README.md b/examples/prometheus/README.md new file mode 100644 index 00000000..77e8a815 --- /dev/null +++ b/examples/prometheus/README.md @@ -0,0 +1,81 @@ +# Prometheus + Alertmanager deployable artifacts + +Drop-in Prometheus configuration for a Switchyard deployment. Pair with +[`docs/METRICS_REFERENCE.md`](../../docs/METRICS_REFERENCE.md) for the +full metric inventory and label semantics. + +## Files + +| File | Purpose | +|---|---| +| `prometheus.yml` | A single `scrape_configs:` entry to merge into your existing Prometheus config. Targets Switchyard's `/metrics`. | +| `switchyard.rules.yaml` | Alert rule group implementing the operational and success-criterion alerts. Validates with `promtool check rules`. | + +A Grafana dashboard is intentionally not included — dashboard authoring +belongs to whichever team owns observability conventions in your +deployment. The metric catalog in `docs/METRICS_REFERENCE.md` is the +input. + +## Wire-up + +### 1. Add the scrape job + +Merge the `scrape_configs:` entry from `prometheus.yml` into your +Prometheus configuration (adjust the `targets:` list to point at your +Switchyard pods or hosts). The endpoint is plain `GET /metrics` over +HTTP, no auth. + +If you're running Switchyard on Kubernetes via +[`examples/k8s/latency-aware-deployment.yaml`](../k8s/latency-aware-deployment.yaml), +the service-level DNS is +`switchyard-latency.llm-routing.svc.cluster.local`. The example scrape +job uses static targets; a `kubernetes_sd_configs` variant is a one-step +swap for cluster-native discovery. + +### 2. Load the alert rules + +```bash +cp switchyard.rules.yaml /etc/prometheus/rules/ +promtool check rules /etc/prometheus/rules/switchyard.rules.yaml +``` + +Then reference the file in your Prometheus config's `rule_files:` +section and reload: + +```yaml +rule_files: + - /etc/prometheus/rules/switchyard.rules.yaml +``` + +```bash +curl -X POST http://:9090/-/reload +``` + +### 3. Validate + +Once Prometheus has scraped a few cycles, sanity-check the active +series: + +```promql +{__name__=~"switchyard_.*"} +``` + +You should see the families documented in `METRICS_REFERENCE.md` — +top-line gauges, per-endpoint counters/summaries, Latency Service state, +poll health, and outcome counters. + +## Alert tuning notes + +Default thresholds and `for:` windows in `switchyard.rules.yaml` are +conservative — tuned to avoid pages during routine scrape-config rolls +or single-poll blips. Review them against your SLOs before enabling +notifications: + +* `LatencyServiceUnreachable` fires after 60s of `poll_ok=0`. Bring this + down if your routing logic should react faster than one poll cycle. +* `RouterOverheadHigh` is set at the success-criterion threshold (p99 > + 1 ms for 5 m). Retry-heavy workloads inflate this — cross-check + `switchyard_upstream_attempts_total{outcome="retryable_error"}` rate + before treating it as a routing-decision regression. +* `RouterNotRescuing` requires upstream failures to be occurring before + it fires, so it does not flap on a healthy chain. diff --git a/examples/prometheus/prometheus.yml b/examples/prometheus/prometheus.yml new file mode 100644 index 00000000..97907247 --- /dev/null +++ b/examples/prometheus/prometheus.yml @@ -0,0 +1,35 @@ +# Example Prometheus scrape configuration for Switchyard. +# +# Merge the `scrape_configs:` entry below into your existing +# prometheus.yml. The `/metrics` endpoint is plain HTTP, no auth, and +# uses the Prometheus text exposition format 0.0.4. +# +# See docs/METRICS_REFERENCE.md for the full metric catalog and +# examples/prometheus/switchyard.rules.yaml for ready-to-load alert +# rules. + +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: switchyard + metrics_path: /metrics + scheme: http + # Default scrape every 15s — the Latency Service poll cycle defaults + # to 10s, so finer scrape resolution captures no additional state. + scrape_interval: 15s + static_configs: + # Replace with your Switchyard host(s) and port(s). On Kubernetes + # with the manifest at examples/k8s/latency-aware-deployment.yaml + # the in-cluster DNS resolves to + # switchyard-latency.llm-routing.svc.cluster.local on port 80 + # targeting container port 4100. Swap this static_configs block + # for kubernetes_sd_configs in a cluster-native deployment. + - targets: + - switchyard.example.internal:4100 + +# Reference the alert rule file shipped alongside this scrape config. +# Adjust the path to wherever your Prometheus loads rule files from. +rule_files: + - switchyard.rules.yaml diff --git a/examples/prometheus/switchyard.rules.yaml b/examples/prometheus/switchyard.rules.yaml new file mode 100644 index 00000000..0f6bc91d --- /dev/null +++ b/examples/prometheus/switchyard.rules.yaml @@ -0,0 +1,130 @@ +# Prometheus alert rules for a Switchyard deployment. +# +# Implements the operational and success-criterion alerts documented in +# docs/METRICS_REFERENCE.md. Drop into Prometheus's rule_files: list and +# validate with `promtool check rules switchyard.rules.yaml` before +# reloading. +# +# Thresholds are conservative defaults. Review against your SLOs in +# examples/prometheus/README.md before wiring to a pager. + +groups: + - name: switchyard.latency_service.health + rules: + - alert: LatencyServiceUnreachable + expr: switchyard_latency_service_poll_ok == 0 + for: 60s + labels: + severity: warning + component: switchyard + annotations: + summary: "Switchyard cannot reach the Latency Service" + description: | + switchyard_latency_service_poll_ok has been 0 for at least + 60s. Routing has fallen back to uniform random across all + endpoints (every verdict reset to unknown). Check the + Switchyard pod logs for "Health poller: failed to reach + Latency Service" and confirm DNS / network reachability to + the upstream LS. + + - alert: LatencyServicePollStale + expr: switchyard_latency_service_poll_age_seconds > 60 + for: 60s + labels: + severity: warning + component: switchyard + annotations: + summary: "Switchyard health verdicts are stale" + description: | + The most recent successful Latency Service poll was more + than 60 s ago. Routing decisions are still using cached + verdicts but they are increasingly out of date. Compare + switchyard_latency_service_poll_failures_total against + switchyard_latency_service_polls_total to see whether the + poller is failing intermittently or hanging entirely. + + - alert: LatencyServiceEndpointFlapping + expr: | + sum by (model) ( + changes(switchyard_endpoint_status{status="healthy"}[10m]) + ) > 5 + for: 0s + labels: + severity: info + component: switchyard + annotations: + summary: "Endpoint {{ $labels.model }} is flapping" + description: | + switchyard_endpoint_status{model="{{ $labels.model }}"} has + transitioned in or out of "healthy" more than 5 times in + the last 10 minutes. Likely a real upstream instability — + cross-check switchyard_endpoint_last_latency_ms for that + model and the LS upstream's own dashboards. + + - name: switchyard.routing.evidence + rules: + - alert: RouterNotRescuing + # Upstream is throwing retryable errors but the router isn't + # absorbing them via retry. This fires only when there IS + # something to rescue — does not flap on a healthy chain. + expr: | + rate(switchyard_router_retry_recovered_total[5m]) == 0 + and + rate(switchyard_upstream_attempts_total{outcome="retryable_error"}[5m]) > 0 + for: 5m + labels: + severity: warning + component: switchyard + annotations: + summary: "Switchyard observed upstream errors but no retry recoveries" + description: | + Upstream attempts are returning retryable errors (429 / 500 + / 504) but no request has been rescued by retry in the same + 5 m window. Likely cause: every configured endpoint is + degraded simultaneously, so the retry loop has nowhere to + steer. Check switchyard_endpoint_status by model. + + - alert: RouterOverheadHigh + expr: switchyard_routing_overhead_ms{quantile="0.99"} > 1.0 + for: 5m + labels: + severity: warning + component: switchyard + annotations: + summary: "Switchyard router overhead p99 above 1 ms" + description: | + switchyard_routing_overhead_ms p99 has exceeded 1 ms for 5 + minutes. Routing overhead bundles format translation, + endpoint selection, body mutation, and retry wall time — + retries are the most common cause of inflation. Cross-check + rate(switchyard_upstream_attempts_total{outcome="retryable_error"}[5m]). + + - alert: RouterErrorRateNotReducing + # Headline success criterion. The router should serve a lower + # rate of retryable errors than what raw upstream attempts + # would produce. If router_error_rate >= direct_error_rate the + # router is not adding the value it claims to. + expr: | + ( + sum(rate(switchyard_client_responses_total{outcome="retryable_error"}[5m])) + / clamp_min(sum(rate(switchyard_client_responses_total[5m])), 1) + ) + >= + ( + sum(rate(switchyard_upstream_attempts_total{outcome="retryable_error"}[5m])) + / clamp_min(sum(rate(switchyard_upstream_attempts_total[5m])), 1) + ) + for: 10m + labels: + severity: warning + component: switchyard + annotations: + summary: "Switchyard router is not reducing client-visible error rate" + description: | + Over the last 5 m, the rate of retryable_error responses to + clients is at least as high as the per-attempt upstream + retryable_error rate. The router is not absorbing failures + via retry — either retries are exhausting or every + endpoint is producing the same failure mode. Look at + switchyard_endpoint_status and the per-model breakdown of + switchyard_errors_total{model}. From 9d8dface8361cb95d15b2ff140818d857ca147dd Mon Sep 17 00:00:00 2001 From: zengyuanl Date: Tue, 7 Jul 2026 19:36:12 +0000 Subject: [PATCH 07/11] chore(secrets): allowlist ported test fixture and refresh baseline --- .secrets.baseline | 22 +++++++++------------- tests/test_request_metadata.py | 2 +- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 8c63cb15..61fde5cb 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -98,10 +98,6 @@ "path": "detect_secrets.filters.common.is_baseline_file", "filename": ".secrets.baseline" }, - { - "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", - "min_level": 2 - }, { "path": "detect_secrets.filters.heuristic.is_indirect_reference" }, @@ -387,7 +383,7 @@ "filename": "switchyard/lib/proxy_context.py", "hashed_secret": "70ca3ed5e568a3c9180bbb4496f1a2072e4ef36f", "is_verified": false, - "line_number": 42 + "line_number": 43 } ], "switchyard/lib/request_metadata.py": [ @@ -593,35 +589,35 @@ "filename": "tests/test_latency_service_llm_backend.py", "hashed_secret": "3acfb2c2b433c0ea7ff107e33df91b18e52f960f", "is_verified": false, - "line_number": 58 + "line_number": 63 }, { "type": "Secret Keyword", "filename": "tests/test_latency_service_llm_backend.py", "hashed_secret": "35188bb026d29646649697a020aa4803df869146", "is_verified": false, - "line_number": 869 + "line_number": 947 }, { "type": "Secret Keyword", "filename": "tests/test_latency_service_llm_backend.py", "hashed_secret": "8294c1af1a5c47187ddf7f15c6fd13f0d4619bec", "is_verified": false, - "line_number": 931 + "line_number": 1009 }, { "type": "Secret Keyword", "filename": "tests/test_latency_service_llm_backend.py", "hashed_secret": "3922bcd29ff79bae723278599532a5cfd75e9b9f", "is_verified": false, - "line_number": 1090 + "line_number": 1219 }, { "type": "Secret Keyword", "filename": "tests/test_latency_service_llm_backend.py", "hashed_secret": "d50bd504c3d7f5dae7d6bf0f06e18280954ba408", "is_verified": false, - "line_number": 1165 + "line_number": 1294 } ], "tests/test_latency_service_stats_metrics.py": [ @@ -761,7 +757,7 @@ "filename": "tests/test_llm_client.py", "hashed_secret": "c77a24bfec49c5b2e2b22fab5369ff079fa4431a", "is_verified": false, - "line_number": 53 + "line_number": 63 } ], "tests/test_outcome_metrics.py": [ @@ -1101,7 +1097,7 @@ "filename": "tests/test_upstream_error_passthrough.py", "hashed_secret": "48854c46907b8f99e4c4c711f3fb7336a93bc290", "is_verified": false, - "line_number": 91 + "line_number": 95 } ], "tests/test_user_config.py": [ @@ -1193,5 +1189,5 @@ } ] }, - "generated_at": "2026-07-06T23:24:20Z" + "generated_at": "2026-07-07T19:36:02Z" } diff --git a/tests/test_request_metadata.py b/tests/test_request_metadata.py index 051c931f..76a9dd44 100644 --- a/tests/test_request_metadata.py +++ b/tests/test_request_metadata.py @@ -123,7 +123,7 @@ def test_key_extracted_but_redacted_in_stored_headers(self) -> None: attach_caller_api_key(ctx, headers) # Extracted for upstream forwarding... - assert ctx.metadata[CTX_CALLER_API_KEY] == "nvapi-secret" + assert ctx.metadata[CTX_CALLER_API_KEY] == "nvapi-secret" # pragma: allowlist secret # ...but the retained header map carries no raw credential. stored = ctx.metadata[CTX_PROFILE_REQUEST_HEADERS] assert stored["x-switchyard-api-key"] == "[REDACTED]" From 3a21cda8778e5741931e9b67886337f9deb86498 Mon Sep 17 00:00:00 2001 From: zengyuanl Date: Tue, 7 Jul 2026 20:00:34 +0000 Subject: [PATCH 08/11] chore: strip issue-tracker references from comments and docs --- docs/internal/latency_service_routing.md | 4 ++-- examples/latency_service_caller_required.yaml | 2 +- switchyard/lib/config/latency_service_backend_config.py | 2 +- tests/test_error_source_annotation.py | 2 +- tests/test_latency_service_llm_backend.py | 4 ++-- tests/test_latency_service_spans.py | 4 ++-- tests/test_llm_client.py | 6 +++--- tests/test_response_translation_engine.py | 2 +- tests/test_upstream_error_passthrough.py | 8 ++++---- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/internal/latency_service_routing.md b/docs/internal/latency_service_routing.md index d6b460e0..9df19dfb 100644 --- a/docs/internal/latency_service_routing.md +++ b/docs/internal/latency_service_routing.md @@ -407,8 +407,8 @@ Wire-level proofs live in (`test_responses_body_returned_exactly_on_the_wire`, `test_responses_stream_forwarded_verbatim_on_the_wire`, `test_azure_flavored_responses_stream_preserved_on_the_wire`) against -OpenAI-shaped and Azure-shaped samples; validation against live Inference Hub -captures remains an IH-side step. +OpenAI-shaped and Azure-shaped samples; validation against live production +captures remains a deployment-side step. ## Failure-source annotation diff --git a/examples/latency_service_caller_required.yaml b/examples/latency_service_caller_required.yaml index 6a829514..3d6f78cf 100644 --- a/examples/latency_service_caller_required.yaml +++ b/examples/latency_service_caller_required.yaml @@ -3,7 +3,7 @@ # ============================================================================= # # Use this when every upstream call must be billed to the *caller's* own API key -# (e.g. per-user Inference Hub spend attribution) and the service-level key must +# (e.g. per-user spend attribution in multi-tenant gateways) and the service-level key must # NEVER authenticate caller inference. # # What `credential_policy: caller_required` does: diff --git a/switchyard/lib/config/latency_service_backend_config.py b/switchyard/lib/config/latency_service_backend_config.py index f2d93ed5..f52da2c3 100644 --- a/switchyard/lib/config/latency_service_backend_config.py +++ b/switchyard/lib/config/latency_service_backend_config.py @@ -102,7 +102,7 @@ class LatencyServiceBackendConfig(BaseModel): forwards the caller key but never falls back — a request with no caller key is rejected with HTTP 401 and the configured ``api_key`` is never used for upstream inference (use this for per-user spend attribution, - e.g. the ``nvidia/switchyard/*`` Inference Hub routes). + e.g. multi-tenant gateway routes). session_affinity: When ``True``, pin each conversation to the endpoint that first served it (cache stays warm); a pin is broken only when its endpoint degrades or the call fails. Per process. Default off. diff --git a/tests/test_error_source_annotation.py b/tests/test_error_source_annotation.py index cba04f8c..6613fd25 100644 --- a/tests/test_error_source_annotation.py +++ b/tests/test_error_source_annotation.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""SWITCH-882: failure-source annotation on error responses and the event log. +"""failure-source annotation on error responses and the event log. Every client-facing error carries ``x-switchyard-error-source`` naming the layer that originated it (``switchyard`` | ``provider``), plus diff --git a/tests/test_latency_service_llm_backend.py b/tests/test_latency_service_llm_backend.py index f22f2617..f97e7c0d 100644 --- a/tests/test_latency_service_llm_backend.py +++ b/tests/test_latency_service_llm_backend.py @@ -834,7 +834,7 @@ async def test_responses_endpoint_preserves_exact_upstream_body(self): """Same-format Responses passthrough returns the upstream JSON untouched. Provider-specific extras and explicit-null fields must survive the - wrap into ``ChatResponse`` (SWITCH-883) — nothing may be normalized, + wrap into ``ChatResponse`` — nothing may be normalized, re-synthesized, or dropped on this path. """ upstream_body = { @@ -1514,7 +1514,7 @@ async def test_metric_absent_until_traffic(self): # --------------------------------------------------------------------------- -# Failure-source annotation (SWITCH-882) +# Failure-source annotation # --------------------------------------------------------------------------- diff --git a/tests/test_latency_service_spans.py b/tests/test_latency_service_spans.py index fd36fe99..09d7be99 100644 --- a/tests/test_latency_service_spans.py +++ b/tests/test_latency_service_spans.py @@ -203,7 +203,7 @@ async def test_api_status_error_tags(tracer: _FakeTracer) -> None: assert attempt.tags["switchyard.upstream_status_code"] == 429 assert attempt.tags["switchyard.outcome"] == "retryable_error" assert attempt.tags["switchyard.error_code"] == "429" - # Failure-source annotation (SWITCH-882): the attempt failed upstream. + # Failure-source annotation: the attempt failed upstream. assert attempt.tags["switchyard.error_source"] == "provider" assert attempt.tags["switchyard.upstream_model"] == "model-A" @@ -221,7 +221,7 @@ async def test_generic_error_tags(tracer: _FakeTracer) -> None: assert attempt.tags["switchyard.error_code"] == "none" # A non-HTTP failure has no status line. assert "switchyard.upstream_status_code" not in attempt.tags - # A network-level fault is still an upstream-side failure (SWITCH-882). + # A network-level fault is still an upstream-side failure. assert attempt.tags["switchyard.error_source"] == "provider" assert attempt.tags["switchyard.upstream_model"] == "model-A" diff --git a/tests/test_llm_client.py b/tests/test_llm_client.py index b6d51ff9..8369b124 100644 --- a/tests/test_llm_client.py +++ b/tests/test_llm_client.py @@ -51,7 +51,7 @@ def _client_with_spied_options() -> tuple[OpenAILLMClient, MagicMock]: overridden.responses.create = AsyncMock(return_value="responses-overridden") client.async_client.with_options.return_value = overridden # Non-streaming ``aresponses`` fetches the raw HTTP response and - # returns its exact JSON body (SWITCH-883); wire raw spies per client. + # returns its exact JSON body; wire raw spies per client. for target, label in ((client.async_client, "base"), (overridden, "overridden")): raw = MagicMock() raw.http_response.json.return_value = {"src": f"responses-{label}"} @@ -122,7 +122,7 @@ async def _lines(): @respx.mock async def test_aresponses_returns_exact_upstream_json() -> None: - """Non-streaming Responses calls return the upstream body as-is (SWITCH-883). + """Non-streaming Responses calls return the upstream body as-is. Provider-specific extras and explicit-null fields must survive — the SDK typed-model round-trip would normalize the former and ``exclude_none`` @@ -173,7 +173,7 @@ async def test_aresponses_returns_exact_upstream_json() -> None: @respx.mock async def test_aresponses_streaming_yields_verbatim_sse_frames() -> None: """Streaming Responses calls yield the upstream SSE frames byte-for-byte - (SWITCH-883): unknown provider fields, explicit nulls, comment keep-alives, + : unknown provider fields, explicit nulls, comment keep-alives, and event names all survive because no typed-model parse happens. """ respx.post("http://upstream.test/v1/responses").mock( diff --git a/tests/test_response_translation_engine.py b/tests/test_response_translation_engine.py index 7cc7cc06..0bfa72d8 100644 --- a/tests/test_response_translation_engine.py +++ b/tests/test_response_translation_engine.py @@ -354,7 +354,7 @@ async def test_reasoning_stream_events_validate_against_anthropic_sdk(self): # --------------------------------------------------------------------------- -# Raw SSE frame parsing for cross-format stream translation (SWITCH-883) +# Raw SSE frame parsing for cross-format stream translation # --------------------------------------------------------------------------- diff --git a/tests/test_upstream_error_passthrough.py b/tests/test_upstream_error_passthrough.py index 1b717507..e2be77f5 100644 --- a/tests/test_upstream_error_passthrough.py +++ b/tests/test_upstream_error_passthrough.py @@ -715,7 +715,7 @@ def test_anthropic_invalid_role_returns_400( # --------------------------------------------------------------------------- -# Failure-source headers on the wire (SWITCH-882) +# Failure-source headers on the wire # --------------------------------------------------------------------------- # The header tests above the endpoint layer call helpers directly; these prove # the annotation actually survives the real FastAPI handlers + middleware to @@ -854,7 +854,7 @@ def test_caller_required_401_header_labels_switchyard_on_the_wire() -> None: # --------------------------------------------------------------------------- -# Responses API fidelity on the wire (SWITCH-883) +# Responses API fidelity on the wire # --------------------------------------------------------------------------- @@ -862,7 +862,7 @@ def test_responses_body_returned_exactly_on_the_wire() -> None: """A Responses request through the full app returns the upstream JSON byte-for-byte: provider extras and explicit-null fields included. - This is the composed proof for SWITCH-883 — backend raw wrap, the + This is the composed fidelity proof — backend raw wrap, the terminal translation short-circuit, and endpoint serialization together. """ upstream_body = { @@ -955,7 +955,7 @@ async def _frame_iter() -> AsyncIterator[str]: def test_responses_stream_forwarded_verbatim_on_the_wire() -> None: """A same-format streaming Responses request returns the upstream SSE frames byte-for-byte: unknown provider fields, explicit nulls, event - names, and comment keep-alives all survive (SWITCH-883 streaming leg).""" + names, and comment keep-alives all survive (streaming fidelity leg).""" frames = [ ( 'event: response.created\n' From da5aa143890fe1019405c20d56f0bdd1410a999f Mon Sep 17 00:00:00 2001 From: zengyuanl Date: Tue, 7 Jul 2026 20:06:26 +0000 Subject: [PATCH 09/11] chore(types): tolerate optional redis import under strict mypy --- switchyard/lib/redis_pin_store.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/switchyard/lib/redis_pin_store.py b/switchyard/lib/redis_pin_store.py index 66b76e6a..0cc19354 100644 --- a/switchyard/lib/redis_pin_store.py +++ b/switchyard/lib/redis_pin_store.py @@ -62,9 +62,10 @@ def _get_client(self) -> Any: # Lazy import keeps ``redis`` an optional dependency. from redis.asyncio import from_url - # redis-py's from_url is untyped; keep strict mypy green for devs - # who install the optional extra. - self._client = from_url( # type: ignore[no-untyped-call] + # redis-py's from_url is untyped when the optional extra is + # installed; without it the import is opaque and the ignore is + # unused — suppress both cases so strict mypy stays green either way. + self._client = from_url( # type: ignore[no-untyped-call,unused-ignore] self._url, decode_responses=True, socket_timeout=self._socket_timeout, From 8a7dce6e93093a05addbf2f6b518b1b690873934 Mon Sep 17 00:00:00 2001 From: zengyuanl Date: Wed, 8 Jul 2026 09:02:30 +0000 Subject: [PATCH 10/11] fix(review): doc-path and terminology fixes, stale-alert timing, early caller_required rejection, shared error envelope --- docs/internal/metrics_reference.md | 2 +- examples/prometheus/README.md | 6 ++--- examples/prometheus/prometheus.yml | 2 +- examples/prometheus/switchyard.rules.yaml | 6 +++-- .../backends/latency_service_llm_backend.py | 25 ++++++++++------- switchyard/lib/endpoints/upstream_error.py | 18 +++++-------- tests/test_latency_service_llm_backend.py | 27 ++++++++++++++++++- 7 files changed, 58 insertions(+), 28 deletions(-) diff --git a/docs/internal/metrics_reference.md b/docs/internal/metrics_reference.md index f2281b5d..859c3a9f 100644 --- a/docs/internal/metrics_reference.md +++ b/docs/internal/metrics_reference.md @@ -63,7 +63,7 @@ Each summary emits `{quantile="0.5"}` and `{quantile="0.99"}` rows plus | Metric | Type | Meaning | |---|---|---| -| `switchyard_routing_overhead_ms{quantile}` | summary | `total_latency − backend_latency`, across all calls. Bundles format translation, endpoint selection, body mutation, response wrapping, and retry wall time. | +| `switchyard_routing_overhead_ms{quantile}` | summary | `total_latency_ms − backend_call_latency_ms`, across all calls. Bundles format translation, endpoint selection, body mutation, response wrapping, and retry wall time. | Healthy traffic typically sits at p50 ≈ 0.4 ms, p99 ≈ 0.6 ms. diff --git a/examples/prometheus/README.md b/examples/prometheus/README.md index 77e8a815..77311cab 100644 --- a/examples/prometheus/README.md +++ b/examples/prometheus/README.md @@ -1,7 +1,7 @@ # Prometheus + Alertmanager deployable artifacts Drop-in Prometheus configuration for a Switchyard deployment. Pair with -[`docs/METRICS_REFERENCE.md`](../../docs/METRICS_REFERENCE.md) for the +[`docs/internal/metrics_reference.md`](../../docs/internal/metrics_reference.md) for the full metric inventory and label semantics. ## Files @@ -13,7 +13,7 @@ full metric inventory and label semantics. A Grafana dashboard is intentionally not included — dashboard authoring belongs to whichever team owns observability conventions in your -deployment. The metric catalog in `docs/METRICS_REFERENCE.md` is the +deployment. The metric catalog in `docs/internal/metrics_reference.md` is the input. ## Wire-up @@ -60,7 +60,7 @@ series: {__name__=~"switchyard_.*"} ``` -You should see the families documented in `METRICS_REFERENCE.md` — +You should see the families documented in `metrics_reference.md` — top-line gauges, per-endpoint counters/summaries, Latency Service state, poll health, and outcome counters. diff --git a/examples/prometheus/prometheus.yml b/examples/prometheus/prometheus.yml index 97907247..229a0831 100644 --- a/examples/prometheus/prometheus.yml +++ b/examples/prometheus/prometheus.yml @@ -4,7 +4,7 @@ # prometheus.yml. The `/metrics` endpoint is plain HTTP, no auth, and # uses the Prometheus text exposition format 0.0.4. # -# See docs/METRICS_REFERENCE.md for the full metric catalog and +# See docs/internal/metrics_reference.md for the full metric catalog and # examples/prometheus/switchyard.rules.yaml for ready-to-load alert # rules. diff --git a/examples/prometheus/switchyard.rules.yaml b/examples/prometheus/switchyard.rules.yaml index 0f6bc91d..dfc8378f 100644 --- a/examples/prometheus/switchyard.rules.yaml +++ b/examples/prometheus/switchyard.rules.yaml @@ -1,7 +1,7 @@ # Prometheus alert rules for a Switchyard deployment. # # Implements the operational and success-criterion alerts documented in -# docs/METRICS_REFERENCE.md. Drop into Prometheus's rule_files: list and +# docs/internal/metrics_reference.md. Drop into Prometheus's rule_files: list and # validate with `promtool check rules switchyard.rules.yaml` before # reloading. # @@ -28,8 +28,10 @@ groups: the upstream LS. - alert: LatencyServicePollStale + # No `for:` — the age gauge only grows while polls keep failing, so + # the >60s threshold is already debounced; a `for:` would only delay + # the alert past the documented 60s staleness budget. expr: switchyard_latency_service_poll_age_seconds > 60 - for: 60s labels: severity: warning component: switchyard diff --git a/switchyard/lib/backends/latency_service_llm_backend.py b/switchyard/lib/backends/latency_service_llm_backend.py index 7f5f96eb..fa8f573c 100644 --- a/switchyard/lib/backends/latency_service_llm_backend.py +++ b/switchyard/lib/backends/latency_service_llm_backend.py @@ -361,6 +361,22 @@ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: # request so the endpoint-layer fallback in ``dispatch`` / # ``handle_chain_exception`` does not double-count the retry fan-out. ctx.metadata[CTX_UPSTREAM_ATTEMPTS_RECORDED] = True + api_key_override = self._api_key_override_for_policy( + ctx.metadata.get(CTX_CALLER_API_KEY) + ) + # ``caller_required`` never falls back to the configured endpoint key: + # reject before any upstream call so the service credential can't + # authenticate caller inference — and before the affinity lookup, so a + # doomed request never pays a pin resolution (possibly a shared-store + # round-trip). The rejection deliberately stays invisible to + # ``switchyard_upstream_attempts_total`` (the accounting flag above is + # already claimed): no upstream attempt happened, and counting a + # synthetic 401 would skew the direct-to-endpoint baseline error rate. + # ``api_key_override`` is ``None`` here only when no usable caller key + # was supplied. + if self._config.credential_policy == "caller_required" and api_key_override is None: + _reject_missing_caller_api_key(ctx) + # Captured before the per-attempt ``body["model"]`` override so the span # records the model the client asked for, not the selected endpoint. incoming_model = request.model @@ -375,15 +391,6 @@ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: # provider-originated failure came from. last_upstream_model: str | None = None tried: set[str] = set() - api_key_override = self._api_key_override_for_policy( - ctx.metadata.get(CTX_CALLER_API_KEY) - ) - # ``caller_required`` never falls back to the configured endpoint key: - # reject before any upstream call so the service credential can't - # authenticate caller inference. ``api_key_override`` is ``None`` here - # only when no usable caller key was supplied. - if self._config.credential_policy == "caller_required" and api_key_override is None: - _reject_missing_caller_api_key(ctx) for attempt in range(1 + self._config.max_retries): # -- Route decision span: which endpoint, out of which candidates -- diff --git a/switchyard/lib/endpoints/upstream_error.py b/switchyard/lib/endpoints/upstream_error.py index f6015a3c..d51b39d4 100644 --- a/switchyard/lib/endpoints/upstream_error.py +++ b/switchyard/lib/endpoints/upstream_error.py @@ -20,7 +20,6 @@ from switchyard.lib.endpoints import outcome_metrics from switchyard.lib.endpoints.error_envelope import ( - ERROR_SOURCE_HEADER, error_response, upstream_error_response, ) @@ -215,14 +214,11 @@ def context_exhausted_response(exc: BaseException, inbound: Inbound) -> JSONResp after consecutive context-window overflows; FastAPI endpoints catch it and call this helper to produce the shared Switchyard HTTP error envelope. """ - error: dict[str, object] = { - "type": "invalid_request_error", - "message": str(exc), - "code": "context_length_exceeded", - } - return JSONResponse( - status_code=400, - content={"error": error}, - # Chain-executor rejection, not an upstream failure. - headers={ERROR_SOURCE_HEADER: ERROR_SOURCE_SWITCHYARD}, + # Chain-executor rejection, not an upstream failure — ``error_response`` + # stamps the ``switchyard`` source header by default. + return error_response( + 400, + str(exc), + error_type="invalid_request_error", + code="context_length_exceeded", ) diff --git a/tests/test_latency_service_llm_backend.py b/tests/test_latency_service_llm_backend.py index f97e7c0d..6b6d0c40 100644 --- a/tests/test_latency_service_llm_backend.py +++ b/tests/test_latency_service_llm_backend.py @@ -1116,7 +1116,10 @@ async def test_caller_required_policy_passes_caller_key_to_chat_sdk(self): async def test_caller_required_policy_rejects_missing_caller_key(self): """No caller key under caller_required → 401 stashed, no upstream call.""" - from switchyard.lib.proxy_context import CTX_UPSTREAM_HTTP_STATUS + from switchyard.lib.proxy_context import ( + CTX_UPSTREAM_ATTEMPTS_RECORDED, + CTX_UPSTREAM_HTTP_STATUS, + ) backend = _make_backend(_config("model-A", credential_policy="caller_required")) backend._clients["model-A"].acompletion = AsyncMock( @@ -1129,11 +1132,19 @@ async def test_caller_required_policy_rejects_missing_caller_key(self): assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401 backend._clients["model-A"].acompletion.assert_not_called() + # Attempt accounting stays claimed: no upstream attempt happened, so + # the endpoint fallback must not count this 401 in + # ``switchyard_upstream_attempts_total``. + assert ctx.metadata[CTX_UPSTREAM_ATTEMPTS_RECORDED] is True + assert "switchyard_latency_upstream_attempts_total" not in "\n".join( + backend._render_prometheus_lines() + ) async def test_caller_required_policy_rejects_blank_caller_key(self): """A blank caller key is unusable → 401, never the configured service key.""" from switchyard.lib.proxy_context import ( CTX_CALLER_API_KEY, + CTX_UPSTREAM_ATTEMPTS_RECORDED, CTX_UPSTREAM_HTTP_STATUS, ) @@ -1149,6 +1160,20 @@ async def test_caller_required_policy_rejects_blank_caller_key(self): assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401 backend._clients["model-A"].acompletion.assert_not_called() + assert ctx.metadata[CTX_UPSTREAM_ATTEMPTS_RECORDED] is True + + async def test_caller_required_rejection_skips_affinity_lookup(self): + """The credential check runs before pin resolution, so a doomed + request never touches the session-affinity store.""" + backend = _make_backend(_config( + "model-A", credential_policy="caller_required", session_affinity=True, + )) + backend._affinity.pinned = AsyncMock() + + with pytest.raises(PermissionError): + await backend.call(ProxyContext(), _openai_request()) + + backend._affinity.pinned.assert_not_awaited() @pytest.mark.parametrize( From fca5d8a2b5c3c95d3502fa06147970c5fa64f686 Mon Sep 17 00:00:00 2001 From: zengyuanl Date: Wed, 8 Jul 2026 09:06:58 +0000 Subject: [PATCH 11/11] chore(secrets): refresh baseline line numbers --- .secrets.baseline | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 61fde5cb..fc265534 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -610,14 +610,14 @@ "filename": "tests/test_latency_service_llm_backend.py", "hashed_secret": "3922bcd29ff79bae723278599532a5cfd75e9b9f", "is_verified": false, - "line_number": 1219 + "line_number": 1244 }, { "type": "Secret Keyword", "filename": "tests/test_latency_service_llm_backend.py", "hashed_secret": "d50bd504c3d7f5dae7d6bf0f06e18280954ba408", "is_verified": false, - "line_number": 1294 + "line_number": 1319 } ], "tests/test_latency_service_stats_metrics.py": [ @@ -1189,5 +1189,5 @@ } ] }, - "generated_at": "2026-07-07T19:36:02Z" + "generated_at": "2026-07-08T09:06:49Z" }