Skip to content

Set the inference gateway Service to externalTrafficPolicy Cluster #320

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
negz merged 1 commit into from
Jul 7, 2026
Merged

Set the inference gateway Service to externalTrafficPolicy Cluster #320

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 42 additions & 2 deletions functions/compose-serving-stack/function/fn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -727,8 +727,8 @@ def compose_dra_driver(self) -> None:
)

def compose_gateway(self) -> None:
"""Compose the GatewayClass and Gateway on the remote cluster. Gated on
ProviderConfigs being observed."""
"""Compose the gateway namespace, EnvoyProxy, GatewayClass, and Gateway on
the remote cluster. Gated on ProviderConfigs being observed."""
pc_observed = self.provider_configs_observed()
pc = _pc_name(self.xr)

Expand Down Expand Up @@ -756,6 +756,39 @@ def compose_gateway(self) -> None:
),
)

# EnvoyProxy pins the managed LoadBalancer Service's externalTrafficPolicy
# to Cluster. Envoy Gateway defaults it to Local, which some clouds' load
# balancers reject (Nebius returns SyncLoadBalancerFailed and never assigns
# an external IP, leaving the gateway address pending and the cluster
# not-Ready). Cluster is accepted by every cloud the provider runs on
# (GKE, EKS, Nebius); the inference gateway does not need client source-IP
# preservation. The GatewayClass references it via parametersRef below. It
# gets no teardown Usage (unlike the Gateway/GatewayClass, which carry Envoy
# Gateway finalizers): it is a plain config object referenced only by
# parametersRef. If Envoy Gateway is found to finalize a referenced
# EnvoyProxy, give it a Usage protected-by gateway-class (see compose_usages).
if pc_observed or "gateway-proxy" in self.req.observed.resources:
resource.update(
self.rsp.desired.resources["gateway-proxy"],
_k8s_object(
pc,
{
"apiVersion": "gateway.envoyproxy.io/v1alpha1",
"kind": "EnvoyProxy",
"metadata": {"name": "inference-gateway", "namespace": "modelplane-system"},
"spec": {
"provider": {
"type": "Kubernetes",
"kubernetes": {
"envoyService": {"externalTrafficPolicy": "Cluster"},
},
},
},
},
metadata=metav1.ObjectMeta(labels={_LABEL_RESOURCE: "gateway-proxy"}),
),
)

if pc_observed or "gateway-class" in self.req.observed.resources:
resource.update(
self.rsp.desired.resources["gateway-class"],
Expand All @@ -767,6 +800,12 @@ def compose_gateway(self) -> None:
"metadata": {"name": gw.className},
"spec": {
"controllerName": "gateway.envoyproxy.io/gatewayclass-controller",
"parametersRef": {
"group": "gateway.envoyproxy.io",
"kind": "EnvoyProxy",
"name": "inference-gateway",
"namespace": "modelplane-system",
},
},
},
metadata=metav1.ObjectMeta(labels={_LABEL_RESOURCE: "gateway-class"}),
Expand Down Expand Up @@ -848,6 +887,7 @@ def mark_readiness(self) -> None:
"dra-driver",
"dra-driver-critical-pods-quota",
"gateway-namespace",
"gateway-proxy",
"gateway-class",
"gateway",
]
Expand Down
92 changes: 92 additions & 0 deletions functions/compose-serving-stack/tests/test_fn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,35 @@ def _gaie_crd_observed() -> dict:
},
}

_GATEWAY_PROXY = {
"apiVersion": "kubernetes.m.crossplane.io/v1alpha1",
"kind": "Object",
"metadata": {
"labels": {"modelplane.ai/resource": "gateway-proxy"},
},
"spec": {
"forProvider": {
"manifest": {
"apiVersion": "gateway.envoyproxy.io/v1alpha1",
"kind": "EnvoyProxy",
"metadata": {"name": "inference-gateway", "namespace": "modelplane-system"},
"spec": {
"provider": {
"type": "Kubernetes",
"kubernetes": {
"envoyService": {"externalTrafficPolicy": "Cluster"},
},
},
},
},
},
"providerConfigRef": {
"kind": "ProviderConfig",
"name": _PC_NAME,
},
},
}

_GATEWAY_CLASS = {
"apiVersion": "kubernetes.m.crossplane.io/v1alpha1",
"kind": "Object",
Expand All @@ -360,6 +389,12 @@ def _gaie_crd_observed() -> dict:
"metadata": {"name": "envoy"},
"spec": {
"controllerName": "gateway.envoyproxy.io/gatewayclass-controller",
"parametersRef": {
"group": "gateway.envoyproxy.io",
"kind": "EnvoyProxy",
"name": "inference-gateway",
"namespace": "modelplane-system",
},
},
},
},
Expand Down Expand Up @@ -655,6 +690,9 @@ async def test_second_pass(self) -> None:
"gateway-namespace": fnv1.Resource(
resource=resource.dict_to_struct(_GATEWAY_NAMESPACE),
),
"gateway-proxy": fnv1.Resource(
resource=resource.dict_to_struct(_GATEWAY_PROXY),
),
"gateway-class": fnv1.Resource(
resource=resource.dict_to_struct(_GATEWAY_CLASS),
),
Expand Down Expand Up @@ -808,6 +846,57 @@ async def test_gateway_gated_on_address(self) -> None:
"gateway address must surface to the XR status once observed",
)

async def test_gateway_proxy_marked_ready_when_observed_ready(self) -> None:
"""The EnvoyProxy (gateway-proxy) is a config-only Object with no runtime
readiness of its own, so it is marked ready from the Ready condition
provider-kubernetes reports on apply, exactly like gateway-namespace and
gateway-class. Left out of mark_readiness it would sit at
READY_UNSPECIFIED and block the ServingStack XR from ever reaching
Ready."""
req = _base_request()
req.observed.resources["provider-config-helm"].CopyFrom(
fnv1.Resource(
resource=resource.dict_to_struct(
{"apiVersion": "helm.m.crossplane.io/v1beta1", "kind": "ProviderConfig"}
),
),
)
req.observed.resources["provider-config-kubernetes"].CopyFrom(
fnv1.Resource(
resource=resource.dict_to_struct(
{"apiVersion": "kubernetes.m.crossplane.io/v1alpha1", "kind": "ProviderConfig"}
),
),
)

# Composed but not yet observed Ready: it must not be marked ready.
got = await self.runner.RunFunction(req, None)
self.assertEqual(
got.desired.resources["gateway-proxy"].ready,
fnv1.READY_UNSPECIFIED,
"gateway-proxy must not be ready before it is observed Ready",
)

# Once provider-kubernetes reports the EnvoyProxy Object Ready=True it is
# marked ready, so it does not hold the composite XR from Ready.
req.observed.resources["gateway-proxy"].CopyFrom(
fnv1.Resource(
resource=resource.dict_to_struct(
{
"apiVersion": "kubernetes.m.crossplane.io/v1alpha1",
"kind": "Object",
"status": {"conditions": [{"type": "Ready", "status": "True"}]},
}
),
),
)
got = await self.runner.RunFunction(req, None)
self.assertEqual(
got.desired.resources["gateway-proxy"].ready,
fnv1.READY_TRUE,
"gateway-proxy must be ready once provider-kubernetes observes it Ready",
)

async def test_third_pass(self) -> None:
"""Steady state: composed releases report Ready, and the gateway address is
surfaced from the observed Object's manifest. The observed gateway Object
Expand Down Expand Up @@ -891,6 +980,9 @@ async def test_third_pass(self) -> None:
"gateway-namespace": fnv1.Resource(
resource=resource.dict_to_struct(_GATEWAY_NAMESPACE),
),
"gateway-proxy": fnv1.Resource(
resource=resource.dict_to_struct(_GATEWAY_PROXY),
),
"gateway-class": fnv1.Resource(
resource=resource.dict_to_struct(_GATEWAY_CLASS),
),
Expand Down
Loading