From 7bd5657f0888137f4e6b2e6936166b9494cd6bed Mon Sep 17 00:00:00 2001 From: Pablo Luna Date: Mon, 6 Jul 2026 08:51:22 -0700 Subject: [PATCH] Set the inference gateway Service to externalTrafficPolicy Cluster Envoy Gateway defaults the managed LoadBalancer Service to externalTrafficPolicy: Local. Some clouds' load balancers reject that: on Nebius the Service emits SyncLoadBalancerFailed and never gets an external IP, so the gateway address stays pending, the ServingStack and InferenceCluster never reach Ready, and deployments targeting the cluster sit unscheduled. It was previously worked around by manually patching the Service. Compose an EnvoyProxy pinning envoyService.externalTrafficPolicy to Cluster and reference it from the GatewayClass via parametersRef, the idiomatic Envoy Gateway way to configure the managed Service. Mark the EnvoyProxy Object ready alongside the other gateway resources so it does not hold the composite from Ready. Cluster is accepted by every cloud the provider runs on (GKE, EKS, Nebius) and the inference gateway does not need client source-IP preservation. Updates the golden tests. Fixes #316. Signed-off-by: Pablo Luna --- .../compose-serving-stack/function/fn.py | 44 ++++++++- .../compose-serving-stack/tests/test_fn.py | 92 +++++++++++++++++++ 2 files changed, 134 insertions(+), 2 deletions(-) diff --git a/functions/compose-serving-stack/function/fn.py b/functions/compose-serving-stack/function/fn.py index 53b30a93..25944e0e 100644 --- a/functions/compose-serving-stack/function/fn.py +++ b/functions/compose-serving-stack/function/fn.py @@ -727,8 +727,8 @@ def compose_dra_driver(self) -> None: ) def compose_gateway(self) -> None: - """Compose the GatewayClass and Gateway on the remote cluster. Gated on - ProviderConfigs being observed.""" + """Compose the gateway namespace, EnvoyProxy, GatewayClass, and Gateway on + the remote cluster. Gated on ProviderConfigs being observed.""" pc_observed = self.provider_configs_observed() pc = _pc_name(self.xr) @@ -756,6 +756,39 @@ def compose_gateway(self) -> None: ), ) + # EnvoyProxy pins the managed LoadBalancer Service's externalTrafficPolicy + # to Cluster. Envoy Gateway defaults it to Local, which some clouds' load + # balancers reject (Nebius returns SyncLoadBalancerFailed and never assigns + # an external IP, leaving the gateway address pending and the cluster + # not-Ready). Cluster is accepted by every cloud the provider runs on + # (GKE, EKS, Nebius); the inference gateway does not need client source-IP + # preservation. The GatewayClass references it via parametersRef below. It + # gets no teardown Usage (unlike the Gateway/GatewayClass, which carry Envoy + # Gateway finalizers): it is a plain config object referenced only by + # parametersRef. If Envoy Gateway is found to finalize a referenced + # EnvoyProxy, give it a Usage protected-by gateway-class (see compose_usages). + if pc_observed or "gateway-proxy" in self.req.observed.resources: + resource.update( + self.rsp.desired.resources["gateway-proxy"], + _k8s_object( + pc, + { + "apiVersion": "gateway.envoyproxy.io/v1alpha1", + "kind": "EnvoyProxy", + "metadata": {"name": "inference-gateway", "namespace": "modelplane-system"}, + "spec": { + "provider": { + "type": "Kubernetes", + "kubernetes": { + "envoyService": {"externalTrafficPolicy": "Cluster"}, + }, + }, + }, + }, + metadata=metav1.ObjectMeta(labels={_LABEL_RESOURCE: "gateway-proxy"}), + ), + ) + if pc_observed or "gateway-class" in self.req.observed.resources: resource.update( self.rsp.desired.resources["gateway-class"], @@ -767,6 +800,12 @@ def compose_gateway(self) -> None: "metadata": {"name": gw.className}, "spec": { "controllerName": "gateway.envoyproxy.io/gatewayclass-controller", + "parametersRef": { + "group": "gateway.envoyproxy.io", + "kind": "EnvoyProxy", + "name": "inference-gateway", + "namespace": "modelplane-system", + }, }, }, metadata=metav1.ObjectMeta(labels={_LABEL_RESOURCE: "gateway-class"}), @@ -848,6 +887,7 @@ def mark_readiness(self) -> None: "dra-driver", "dra-driver-critical-pods-quota", "gateway-namespace", + "gateway-proxy", "gateway-class", "gateway", ] diff --git a/functions/compose-serving-stack/tests/test_fn.py b/functions/compose-serving-stack/tests/test_fn.py index 4265ecf8..eb35a1ea 100644 --- a/functions/compose-serving-stack/tests/test_fn.py +++ b/functions/compose-serving-stack/tests/test_fn.py @@ -346,6 +346,35 @@ def _gaie_crd_observed() -> dict: }, } +_GATEWAY_PROXY = { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "metadata": { + "labels": {"modelplane.ai/resource": "gateway-proxy"}, + }, + "spec": { + "forProvider": { + "manifest": { + "apiVersion": "gateway.envoyproxy.io/v1alpha1", + "kind": "EnvoyProxy", + "metadata": {"name": "inference-gateway", "namespace": "modelplane-system"}, + "spec": { + "provider": { + "type": "Kubernetes", + "kubernetes": { + "envoyService": {"externalTrafficPolicy": "Cluster"}, + }, + }, + }, + }, + }, + "providerConfigRef": { + "kind": "ProviderConfig", + "name": _PC_NAME, + }, + }, +} + _GATEWAY_CLASS = { "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", "kind": "Object", @@ -360,6 +389,12 @@ def _gaie_crd_observed() -> dict: "metadata": {"name": "envoy"}, "spec": { "controllerName": "gateway.envoyproxy.io/gatewayclass-controller", + "parametersRef": { + "group": "gateway.envoyproxy.io", + "kind": "EnvoyProxy", + "name": "inference-gateway", + "namespace": "modelplane-system", + }, }, }, }, @@ -655,6 +690,9 @@ async def test_second_pass(self) -> None: "gateway-namespace": fnv1.Resource( resource=resource.dict_to_struct(_GATEWAY_NAMESPACE), ), + "gateway-proxy": fnv1.Resource( + resource=resource.dict_to_struct(_GATEWAY_PROXY), + ), "gateway-class": fnv1.Resource( resource=resource.dict_to_struct(_GATEWAY_CLASS), ), @@ -808,6 +846,57 @@ async def test_gateway_gated_on_address(self) -> None: "gateway address must surface to the XR status once observed", ) + async def test_gateway_proxy_marked_ready_when_observed_ready(self) -> None: + """The EnvoyProxy (gateway-proxy) is a config-only Object with no runtime + readiness of its own, so it is marked ready from the Ready condition + provider-kubernetes reports on apply, exactly like gateway-namespace and + gateway-class. Left out of mark_readiness it would sit at + READY_UNSPECIFIED and block the ServingStack XR from ever reaching + Ready.""" + req = _base_request() + req.observed.resources["provider-config-helm"].CopyFrom( + fnv1.Resource( + resource=resource.dict_to_struct( + {"apiVersion": "helm.m.crossplane.io/v1beta1", "kind": "ProviderConfig"} + ), + ), + ) + req.observed.resources["provider-config-kubernetes"].CopyFrom( + fnv1.Resource( + resource=resource.dict_to_struct( + {"apiVersion": "kubernetes.m.crossplane.io/v1alpha1", "kind": "ProviderConfig"} + ), + ), + ) + + # Composed but not yet observed Ready: it must not be marked ready. + got = await self.runner.RunFunction(req, None) + self.assertEqual( + got.desired.resources["gateway-proxy"].ready, + fnv1.READY_UNSPECIFIED, + "gateway-proxy must not be ready before it is observed Ready", + ) + + # Once provider-kubernetes reports the EnvoyProxy Object Ready=True it is + # marked ready, so it does not hold the composite XR from Ready. + req.observed.resources["gateway-proxy"].CopyFrom( + fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "Object", + "status": {"conditions": [{"type": "Ready", "status": "True"}]}, + } + ), + ), + ) + got = await self.runner.RunFunction(req, None) + self.assertEqual( + got.desired.resources["gateway-proxy"].ready, + fnv1.READY_TRUE, + "gateway-proxy must be ready once provider-kubernetes observes it Ready", + ) + async def test_third_pass(self) -> None: """Steady state: composed releases report Ready, and the gateway address is surfaced from the observed Object's manifest. The observed gateway Object @@ -891,6 +980,9 @@ async def test_third_pass(self) -> None: "gateway-namespace": fnv1.Resource( resource=resource.dict_to_struct(_GATEWAY_NAMESPACE), ), + "gateway-proxy": fnv1.Resource( + resource=resource.dict_to_struct(_GATEWAY_PROXY), + ), "gateway-class": fnv1.Resource( resource=resource.dict_to_struct(_GATEWAY_CLASS), ),